Skip to content

Instantly share code, notes, and snippets.

@jmw511
Created April 15, 2015 22:55
Show Gist options
  • Save jmw511/712ac0bb4ce6d07ff605 to your computer and use it in GitHub Desktop.
Save jmw511/712ac0bb4ce6d07ff605 to your computer and use it in GitHub Desktop.
Script to reimport errybody
package main
import (
"encoding/csv"
"fmt"
"io"
"os"
"strconv"
"time"
"github.com/timehop/golog/log"
"github.com/timehop/goth/env"
"github.com/timehop/importers/jobs"
"github.com/timehop/jimmy/redis"
)
var jobTypes = []string{"facebook_statuses", "facebook_uploaded_photos", "facebook_tagged_photos", "facebook_feed"}
type userIDs struct {
userID string
fbUserID string
}
func main() {
importerRedisURL := "redis://localhost:6379"
redisURL := "redis://localhost:6379"
Redis := redis.NewPool(redisURL, redis.DefaultConfig)
ImporterRedis := redis.NewPool(importerRedisURL, redis.DefaultConfig)
ImportDrainRedisKey := "import:drain"
fmt.Println("redis", Redis, "ImporterRedis", ImporterRedis, ImportDrainRedisKey)
csvFilePath := os.Args[1]
log.Info("", "Importing values from file.", "file", csvFilePath)
file, err := os.Open(os.Args[1])
if err != nil {
log.Fatal("", "Could not open file.", "error", err)
return
}
defer file.Close()
skip := env.VarAsInt("SKIP", 0)
limit := env.VarAsInt("LIMIT", 0)
reader := csv.NewReader(file)
total := 0
for {
line, err := reader.Read()
if err == io.EOF {
log.Debug("", "Reached end of CSV file.")
break
} else if err != nil {
log.Error("", "Reached end of CSV file.")
break
} else {
total++
if total < skip {
continue
}
fmt.Println("line", line)
i := userIDs{userID: line[0], fbUserID: line[1]}
for _, jobType := range jobTypes {
userIDInt64, err := strconv.ParseInt(i.userID, 10, 64)
if err != nil {
panic(fmt.Sprintf("Error converting user ID. Last user ID: %v", i.userID))
}
job := jobs.Job{
UserID: userIDInt64,
Type: jobType,
Time: time.Now().Unix(),
IsHighPriority: false,
IsReimport: false,
Params: map[string]string{
"uid": i.fbUserID,
},
}
jsonJob := job.JSON()
fmt.Println("job", jsonJob)
imported, err := ImporterRedis.SAdd(ImportDrainRedisKey, jsonJob)
if err != nil {
panic(fmt.Sprintf("Redis error. Last user ID: %v", i.userID))
}
fmt.Println("imported", imported)
}
if limit > 0 && total >= limit {
break
}
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment