added revalidate cache feature

cmd/steamimmich.go

@@ -17,6 +17,7 @@ type args struct {
 	DeviceID    string `arg:"--device-id" default:"steam-immich" help:"Device ID of the uploads"`
 	Album       string `arg:"--album" help:"UUID of a album to upload to"`
 	CacheFile   string `arg:"--cache" default:"$XDG_CACHE_HOME/steam-immich.json" help:"Location of the cache file"`
+	Revalidate  bool   `args:"--revalidate" default:"false" help:"Only revalidate and fix the cache"`
 }
 
 func Run() {
@@ -38,14 +39,20 @@ func Run() {
 	// Disable timestamp in log
 	log.SetFlags(0)
 
-	steamimmich.Run(steamimmich.Config{
+	config := steamimmich.Config{
 		APIKey:      args.APIKey,
 		BaseURL:     args.BaseURL,
 		UserdataDir: steamUserdataDir,
 		DeiveID:     args.DeviceID,
 		Album:       args.Album,
 		CacheFile:   os.ExpandEnv(args.CacheFile),
-	})
+	}
+
+	if args.Revalidate {
+		steamimmich.Revalidate(config)
+	} else {
+		steamimmich.Run(config)
+	}
 }
 
 func findUserdataDir() string {

internal/hasher.go

@@ -0,0 +1,31 @@
+package steamimmich
+
+import (
+	"crypto/sha1"
+	"fmt"
+	"io"
+	"os"
+)
+
+func hashFiles(assets []AssetState) (map[int]string, error) {
+	results := make(map[int]string)
+
+	for i, p := range assets {
+		f, err := os.Open(p.FilePath)
+		if err != nil {
+			results[i] = ""
+			continue
+		}
+
+		h := sha1.New()
+		if _, err := io.Copy(h, f); err != nil {
+			f.Close()
+			return nil, err
+		}
+		f.Close()
+
+		results[i] = fmt.Sprintf("%x", h.Sum(nil))
+	}
+
+	return results, nil
+}

internal/immich.go

@@ -28,6 +28,19 @@ type bulkIdResponseDto struct {
 	Success bool   `json:"success"`
 }
 
+type assetBulkUploadCheckItem struct {
+	Checksum string `json:"checksum"`
+	Id       string `json:"id"` // Not the assetID. Just a arbitrarily id string
+}
+
+type assetBulkUploadCheckResult struct {
+	Action    string // can be "accept" or "reject"
+	AssetId   string
+	Id        string // Not the assetID. Just a arbitrary id string
+	IsTrashed bool
+	Reason    string // can be "duplicate" or "unsupportedFormat" (i don't know what they mean by this)
+}
+
 type APIKeyTransport struct {
 	APIKey string
 }
@@ -163,3 +176,38 @@ func addAssetsToAlbum(assets []string, album, baseURL string, httpClient http.Cl
 
 	return nil
 }
+
+func bulkCheckAssets(assets []assetBulkUploadCheckItem, baseURL string, httpClient http.Client) ([]assetBulkUploadCheckResult, error) {
+	body, _ := json.Marshal(struct {
+		Assets []assetBulkUploadCheckItem `json:"assets"`
+	}{Assets: assets})
+
+	req, err := http.NewRequest("POST", fmt.Sprintf("%s/api/assets/bulk-upload-check", baseURL), bytes.NewBuffer(body))
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Accept", "application/json")
+
+	res, err := httpClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	defer res.Body.Close()
+
+	respBody, _ := io.ReadAll(res.Body)
+	if res.StatusCode != 200 {
+		return nil, fmt.Errorf("buck check assets %d - %s ", res.StatusCode, string(respBody))
+	}
+
+	var result struct {
+		Results []assetBulkUploadCheckResult `json:"results"`
+	}
+	if err := json.Unmarshal(respBody, &result); err != nil {
+		return nil, fmt.Errorf("parse immich response: %s", err)
+	}
+
+	return result.Results, nil
+}

internal/steamimmich.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"log"
 	"slices"
+	"sort"
+	"strconv"
 )
 
 type Config struct {
@@ -86,3 +88,84 @@ func Run(config Config) {
 		return
 	}
 }
+
+func Revalidate(config Config) {
+	immichClient := newImmichHttpClient(config.APIKey)
+
+	localState, err := loadLocalState(config.CacheFile)
+	if err != nil {
+		log.Fatalf("Failed to load local state: %s", err)
+		return
+	}
+
+	hashes, err := hashFiles(localState.Assets)
+	if err != nil {
+		log.Fatalf("Failed to generate hashes: %s", err)
+		return
+	}
+
+	idxToRemoveFromCache := []int{}
+
+	rejected := 0
+	updated := 0
+	missingFromImmich := 0
+	missingFile := 0
+
+	// Prepare payload
+	var idsToCheck = []assetBulkUploadCheckItem{}
+	for i := range localState.Assets {
+		hash := hashes[i]
+		if hash == "" {
+			idxToRemoveFromCache = append(idxToRemoveFromCache, i)
+			missingFile += 1
+		} else {
+			idsToCheck = append(idsToCheck, assetBulkUploadCheckItem{Checksum: hashes[i], Id: strconv.Itoa(i)})
+		}
+	}
+
+	results, err := bulkCheckAssets(idsToCheck, config.BaseURL, immichClient)
+	if err != nil {
+		log.Fatalf("Failed to bulk check hashes: %s", err)
+	}
+
+	for _, result := range results {
+		i, err := strconv.Atoi(result.Id)
+		if err != nil {
+			log.Fatalf("What ?: %s", err)
+			return
+		}
+
+		asset := localState.Assets[i]
+
+		if asset.ImmichID != result.AssetId {
+			log.Printf("Asset %s had the wrong Immich ID. New %s", asset.FilePath, result.AssetId)
+			localState.Assets[i].ImmichID = result.AssetId
+			updated += 1
+		}
+
+		if result.Action == "reject" {
+			rejected += 1
+		} else {
+			idxToRemoveFromCache = append(idxToRemoveFromCache, i)
+			missingFromImmich += 1
+		}
+	}
+
+	sort.Sort(sort.Reverse(sort.IntSlice(idxToRemoveFromCache)))
+	for _, idx := range idxToRemoveFromCache {
+		localState.Assets[idx] = localState.Assets[len(localState.Assets)-1]
+		localState.Assets = localState.Assets[:len(localState.Assets)-1]
+	}
+
+	saveLocalState(config.CacheFile, *localState)
+
+	log.Printf("Revalidated %d assets", len(results))
+	log.Printf("Assets present on Immich: %d", rejected)
+	log.Printf("Wrong metadata on cache: %d", updated)
+	log.Printf("Missing on Immich: %d", missingFromImmich)
+	log.Printf("Missing on local storage: %d", missingFile)
+
+	if (updated + missingFromImmich + missingFromImmich) > 0 {
+		log.Printf("Fixed cache. Run the normal upload command again. If error persists try removing cache file at: %s", config.CacheFile)
+	}
+}