package cache import ( "crypto/md5" "encoding/hex" "fmt" "io" "log" "mime" "net/http" "net/url" "os" "path" "regexp" "strconv" "strings" "sync" "time" ) var filepattern = regexp.MustCompile(`((In)?Release(\.gpg)?|Packages(\.gz|\.bz2)?|Contents-(arm64|armhf|amd64)(\.gz)|Sources.bz2?)`) // Reader is a generic interface for reading cache entries either from disk or // directly attached to a downloader. type Reader interface { io.ReadCloser GetEntry() (*Entry, error) } // Cache provides access to entries in the cache. type Cache struct { mutex sync.Mutex directory string downloaders map[string]*downloader waitGroup sync.WaitGroup } // NewCache creates a new cache in the specified directory. func NewCache(directory string) (*Cache, error) { if err := os.MkdirAll(directory, 0775); err != nil { return nil, err } return &Cache{ directory: directory, downloaders: make(map[string]*downloader), }, nil } // getFilenames returns the filenames for the JSON and data files from a URL. func getRelURL(rawurl string) (relurl string) { parsed, err := url.Parse(rawurl) if err != nil { return } relurl = parsed.Path return } // getFilenames returns the filenames for the JSON and data files from a URL. func (c *Cache) getFilenames(rawurl string) (hash, jsonFilename, dataFilename string) { b := md5.Sum([]byte(rawurl)) hash = hex.EncodeToString(b[:]) jsonFilename = path.Join(c.directory, fmt.Sprintf("%s.json", hash)) dataFilename = path.Join(c.directory, fmt.Sprintf("%s.data", hash)) return } // GetReader obtains a Reader for the specified rawurl. If a downloader // currently exists for the URL, a live reader is created and connected to it. // If the URL exists in the cache, it is read using the standard file API. If // not, a downloader and live reader are created. func (c *Cache) GetReader(rawurl string, maxAge time.Duration) (Reader, error) { relurl := getRelURL(rawurl) hash, jsonFilename, dataFilename := c.getFilenames(relurl) c.mutex.Lock() defer c.mutex.Unlock() d, ok := c.downloaders[hash] if !ok { _, err := os.Stat(jsonFilename) if err != nil { if !os.IsNotExist(err) { return nil, err } } else { r, err := newDiskReader(jsonFilename, dataFilename) if err != nil { return nil, err } e, _ := r.GetEntry() lastModified, _ := time.Parse(http.TimeFormat, e.LastModified) if e.Complete && !filepattern.MatchString(rawurl) && (maxAge == -1 || lastModified.Before(time.Now().Add(maxAge))) { log.Println("[HIT]", rawurl) return r, nil } } d = newDownloader(rawurl, jsonFilename, dataFilename) go func() { d.WaitForDone() c.mutex.Lock() defer c.mutex.Unlock() delete(c.downloaders, hash) c.waitGroup.Done() }() c.downloaders[hash] = d c.waitGroup.Add(1) } log.Println("[MISS]", rawurl) return newLiveReader(d, dataFilename) } // Insert adds an item into the cache. func (c *Cache) Insert(rawurl string, r io.Reader) error { relurl := getRelURL(rawurl) _, jsonFilename, dataFilename := c.getFilenames(relurl) f, err := os.Open(dataFilename) if err != nil { return err } defer f.Close() n, err := io.Copy(f, r) if err != nil { return err } e := &Entry{ URL: rawurl, RelURL: relurl, Complete: true, ContentLength: strconv.FormatInt(n, 10), ContentType: mime.TypeByExtension(rawurl), LastModified: time.Now().Format(http.TimeFormat), LastAccessed: time.Now().Format(http.TimeFormat), } return e.Save(jsonFilename) } // TODO: implement some form of "safe abort" for downloads so that the entire // application doesn't end up spinning its tires waiting for downloads to end. // Close waits for all downloaders to complete before shutting down. func (c *Cache) Close() { c.waitGroup.Wait() } // Migrate make file migrations with older versions func (c *Cache) Migrate(pathname *string) (err error) { files, err := os.ReadDir(*pathname) for _, f := range files { if strings.HasSuffix(f.Name(), ".json") { basename := strings.Split(f.Name(), ".")[0] jsonfile := fmt.Sprintf("%s/%s.json", *pathname, basename) datafile := fmt.Sprintf("%s/%s.data", *pathname, basename) newDiskReader(jsonfile, datafile) e := Entry{} err = e.Load(jsonfile) if err != nil { log.Println(err) continue } relurl := getRelURL(e.URL) _, newjsonfile, newdatafile := c.getFilenames(relurl) err = os.Rename(jsonfile, newjsonfile) if err != nil { log.Println(err) continue } err = os.Rename(datafile, newdatafile) if err != nil { log.Println(err) continue } } } return }