cmd/speedtest-sync: fail-fast on bad timestamps + minor cleanups

- Skip rows with null/malformed timestamps instead of silently
  re-stamping with time.Now() (data corruption hidden behind a
  fallback). Drops the time import.
- Replace ad-hoc parentDir() with stdlib filepath.Dir().
- Add t.Fatal error checks to TestExtractRows_SkipsBadExtra
  (CREATE TABLE / INSERT) and switch sql.Open to checked form.
- New TestExtractRows_SkipsBadTimestamp covers the null-ts path.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
BergaBruh
2026-05-09 18:53:34 +05:00
parent 367db68837
commit e6f1f63b8d
3 changed files with 53 additions and 21 deletions
+2 -10
View File
@@ -6,6 +6,7 @@ import (
"flag" "flag"
"log" "log"
"os" "os"
"path/filepath"
"strconv" "strconv"
"time" "time"
@@ -99,17 +100,8 @@ func readCursor(path string) int64 {
} }
func writeCursor(path string, id int64) error { func writeCursor(path string, id int64) error {
if err := os.MkdirAll(parentDir(path), 0o755); err != nil { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return err return err
} }
return os.WriteFile(path, []byte(strconv.FormatInt(id, 10)), 0o600) return os.WriteFile(path, []byte(strconv.FormatInt(id, 10)), 0o600)
} }
func parentDir(p string) string {
for i := len(p) - 1; i >= 0; i-- {
if p[i] == '/' {
return p[:i]
}
}
return "."
}
+7 -6
View File
@@ -6,7 +6,6 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"log" "log"
"time"
pb "github.com/bergabruh/stats-gateway/gen/stats/v1" pb "github.com/bergabruh/stats-gateway/gen/stats/v1"
) )
@@ -63,12 +62,14 @@ func extractRows(ctx context.Context, db *sql.DB, sinceID int64) ([]extractedRow
continue continue
} }
var tsUnix int64 if !ts.Valid {
if ts.Valid { log.Printf("skip id=%d: null timestamp", id)
fmt.Sscanf(ts.String, "%d", &tsUnix) continue
} }
if tsUnix == 0 { var tsUnix int64
tsUnix = time.Now().Unix() if _, err := fmt.Sscanf(ts.String, "%d", &tsUnix); err != nil || tsUnix == 0 {
log.Printf("skip id=%d: bad timestamp %q", id, ts.String)
continue
} }
out = append(out, extractedRow{ out = append(out, extractedRow{
+44 -5
View File
@@ -54,15 +54,22 @@ func TestExtractRows_FiltersExtraJSON(t *testing.T) {
func TestExtractRows_SkipsBadExtra(t *testing.T) { func TestExtractRows_SkipsBadExtra(t *testing.T) {
dir := t.TempDir() dir := t.TempDir()
dbPath := filepath.Join(dir, "test.db") dbPath := filepath.Join(dir, "test.db")
db, _ := sql.Open("sqlite", dbPath) db, err := sql.Open("sqlite", dbPath)
if err != nil {
t.Fatal(err)
}
defer db.Close() defer db.Close()
db.Exec(`CREATE TABLE speedtest_users ( if _, err := db.Exec(`CREATE TABLE speedtest_users (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp DATETIME, ip TEXT, ispinfo TEXT, extra TEXT, timestamp DATETIME, ip TEXT, ispinfo TEXT, extra TEXT,
ua TEXT, lang TEXT, dl REAL, ul REAL, ping REAL, jitter REAL, log TEXT)`) ua TEXT, lang TEXT, dl REAL, ul REAL, ping REAL, jitter REAL, log TEXT)`); err != nil {
db.Exec(`INSERT INTO speedtest_users(timestamp, extra, dl, ul, ping, jitter) t.Fatal(err)
VALUES(datetime('now'), 'not-json', 1, 1, 1, 1)`) }
if _, err := db.Exec(`INSERT INTO speedtest_users(timestamp, extra, dl, ul, ping, jitter)
VALUES(datetime('now'), 'not-json', 1, 1, 1, 1)`); err != nil {
t.Fatal(err)
}
rows, err := extractRows(context.Background(), db, 0) rows, err := extractRows(context.Background(), db, 0)
if err != nil { if err != nil {
@@ -72,3 +79,35 @@ func TestExtractRows_SkipsBadExtra(t *testing.T) {
t.Errorf("expected 0 rows (bad extra skipped), got %d", len(rows)) t.Errorf("expected 0 rows (bad extra skipped), got %d", len(rows))
} }
} }
func TestExtractRows_SkipsBadTimestamp(t *testing.T) {
dir := t.TempDir()
dbPath := filepath.Join(dir, "test.db")
db, err := sql.Open("sqlite", dbPath)
if err != nil {
t.Fatal(err)
}
defer db.Close()
if _, err := db.Exec(`CREATE TABLE speedtest_users (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp DATETIME, ip TEXT, ispinfo TEXT, extra TEXT,
ua TEXT, lang TEXT, dl REAL, ul REAL, ping REAL, jitter REAL, log TEXT)`); err != nil {
t.Fatal(err)
}
extra, _ := json.Marshal(map[string]string{"node": "fl", "country": "RU"})
// NULL timestamp
if _, err := db.Exec(`INSERT INTO speedtest_users(timestamp, extra, dl, ul, ping, jitter)
VALUES(NULL, ?, 1, 1, 1, 1)`, string(extra)); err != nil {
t.Fatal(err)
}
rows, err := extractRows(context.Background(), db, 0)
if err != nil {
t.Fatal(err)
}
if len(rows) != 0 {
t.Errorf("expected 0 rows (null timestamp skipped), got %d", len(rows))
}
}