cmd/speedtest-sync: fail-fast on bad timestamps + minor cleanups
- Skip rows with null/malformed timestamps instead of silently re-stamping with time.Now() (data corruption hidden behind a fallback). Drops the time import. - Replace ad-hoc parentDir() with stdlib filepath.Dir(). - Add t.Fatal error checks to TestExtractRows_SkipsBadExtra (CREATE TABLE / INSERT) and switch sql.Open to checked form. - New TestExtractRows_SkipsBadTimestamp covers the null-ts path. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -6,6 +6,7 @@ import (
|
|||||||
"flag"
|
"flag"
|
||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"strconv"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -99,17 +100,8 @@ func readCursor(path string) int64 {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func writeCursor(path string, id int64) error {
|
func writeCursor(path string, id int64) error {
|
||||||
if err := os.MkdirAll(parentDir(path), 0o755); err != nil {
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return os.WriteFile(path, []byte(strconv.FormatInt(id, 10)), 0o600)
|
return os.WriteFile(path, []byte(strconv.FormatInt(id, 10)), 0o600)
|
||||||
}
|
}
|
||||||
|
|
||||||
func parentDir(p string) string {
|
|
||||||
for i := len(p) - 1; i >= 0; i-- {
|
|
||||||
if p[i] == '/' {
|
|
||||||
return p[:i]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return "."
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"time"
|
|
||||||
|
|
||||||
pb "github.com/bergabruh/stats-gateway/gen/stats/v1"
|
pb "github.com/bergabruh/stats-gateway/gen/stats/v1"
|
||||||
)
|
)
|
||||||
@@ -63,12 +62,14 @@ func extractRows(ctx context.Context, db *sql.DB, sinceID int64) ([]extractedRow
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
var tsUnix int64
|
if !ts.Valid {
|
||||||
if ts.Valid {
|
log.Printf("skip id=%d: null timestamp", id)
|
||||||
fmt.Sscanf(ts.String, "%d", &tsUnix)
|
continue
|
||||||
}
|
}
|
||||||
if tsUnix == 0 {
|
var tsUnix int64
|
||||||
tsUnix = time.Now().Unix()
|
if _, err := fmt.Sscanf(ts.String, "%d", &tsUnix); err != nil || tsUnix == 0 {
|
||||||
|
log.Printf("skip id=%d: bad timestamp %q", id, ts.String)
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
out = append(out, extractedRow{
|
out = append(out, extractedRow{
|
||||||
|
|||||||
@@ -54,15 +54,22 @@ func TestExtractRows_FiltersExtraJSON(t *testing.T) {
|
|||||||
func TestExtractRows_SkipsBadExtra(t *testing.T) {
|
func TestExtractRows_SkipsBadExtra(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
dbPath := filepath.Join(dir, "test.db")
|
dbPath := filepath.Join(dir, "test.db")
|
||||||
db, _ := sql.Open("sqlite", dbPath)
|
db, err := sql.Open("sqlite", dbPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
defer db.Close()
|
defer db.Close()
|
||||||
|
|
||||||
db.Exec(`CREATE TABLE speedtest_users (
|
if _, err := db.Exec(`CREATE TABLE speedtest_users (
|
||||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
timestamp DATETIME, ip TEXT, ispinfo TEXT, extra TEXT,
|
timestamp DATETIME, ip TEXT, ispinfo TEXT, extra TEXT,
|
||||||
ua TEXT, lang TEXT, dl REAL, ul REAL, ping REAL, jitter REAL, log TEXT)`)
|
ua TEXT, lang TEXT, dl REAL, ul REAL, ping REAL, jitter REAL, log TEXT)`); err != nil {
|
||||||
db.Exec(`INSERT INTO speedtest_users(timestamp, extra, dl, ul, ping, jitter)
|
t.Fatal(err)
|
||||||
VALUES(datetime('now'), 'not-json', 1, 1, 1, 1)`)
|
}
|
||||||
|
if _, err := db.Exec(`INSERT INTO speedtest_users(timestamp, extra, dl, ul, ping, jitter)
|
||||||
|
VALUES(datetime('now'), 'not-json', 1, 1, 1, 1)`); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
rows, err := extractRows(context.Background(), db, 0)
|
rows, err := extractRows(context.Background(), db, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -72,3 +79,35 @@ func TestExtractRows_SkipsBadExtra(t *testing.T) {
|
|||||||
t.Errorf("expected 0 rows (bad extra skipped), got %d", len(rows))
|
t.Errorf("expected 0 rows (bad extra skipped), got %d", len(rows))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExtractRows_SkipsBadTimestamp(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
dbPath := filepath.Join(dir, "test.db")
|
||||||
|
db, err := sql.Open("sqlite", dbPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
|
||||||
|
if _, err := db.Exec(`CREATE TABLE speedtest_users (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
timestamp DATETIME, ip TEXT, ispinfo TEXT, extra TEXT,
|
||||||
|
ua TEXT, lang TEXT, dl REAL, ul REAL, ping REAL, jitter REAL, log TEXT)`); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
extra, _ := json.Marshal(map[string]string{"node": "fl", "country": "RU"})
|
||||||
|
// NULL timestamp
|
||||||
|
if _, err := db.Exec(`INSERT INTO speedtest_users(timestamp, extra, dl, ul, ping, jitter)
|
||||||
|
VALUES(NULL, ?, 1, 1, 1, 1)`, string(extra)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
rows, err := extractRows(context.Background(), db, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(rows) != 0 {
|
||||||
|
t.Errorf("expected 0 rows (null timestamp skipped), got %d", len(rows))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user