From 7bcb3539e8abe02061ba71800015391ddf2e4d80 Mon Sep 17 00:00:00 2001 From: Nathan Osman Date: Tue, 26 Apr 2016 13:12:46 -0700 Subject: [PATCH] Reworked a lot of the code and added server type. --- cache.go | 95 +++++++++++ downloader.go | 88 ++++++++++ entry.go | 12 +- livereader.go | 96 +++++++++++ main.go | 19 +++ mirrors.json | 396 -------------------------------------------- mirrors/ubuntu.json | 396 -------------------------------------------- reader.go | 126 -------------- server.go | 98 +++++++++++ storage.go | 57 ------- writer.go | 103 ------------ 11 files changed, 404 insertions(+), 1082 deletions(-) create mode 100644 cache.go create mode 100644 downloader.go create mode 100644 livereader.go create mode 100644 main.go delete mode 100644 mirrors.json delete mode 100644 mirrors/ubuntu.json delete mode 100644 reader.go create mode 100644 server.go delete mode 100644 storage.go delete mode 100644 writer.go diff --git a/cache.go b/cache.go new file mode 100644 index 0000000..a477d02 --- /dev/null +++ b/cache.go @@ -0,0 +1,95 @@ +package main + +import ( + "crypto/md5" + "encoding/hex" + "fmt" + "io" + "log" + "os" + "path" + "sync" +) + +// Cache provides access to entries in the cache. +type Cache struct { + mutex sync.Mutex + directory string + downloaders map[string]*Downloader + waitGroup sync.WaitGroup +} + +// NewCache creates a new cache in the specified directory. +func NewCache(directory string) *Cache { + return &Cache{ + directory: directory, + downloaders: make(map[string]*Downloader), + } +} + +// GetReader obtains an io.Reader for the specified rawurl. If a downloader +// currently exists for the URL, a live reader is created and connected to it. +// If the URL exists in the cache, it is read using the standard file API. If +// not, a downloader and live reader are created. +func (c *Cache) GetReader(rawurl string) (io.ReadCloser, chan *Entry, error) { + var ( + b = md5.Sum([]byte(rawurl)) + hash = hex.EncodeToString(b[:]) + jsonFilename = path.Join(c.directory, fmt.Sprintf("%s.json", hash)) + dataFilename = path.Join(c.directory, fmt.Sprintf("%s.data", hash)) + ) + c.mutex.Lock() + defer c.mutex.Unlock() + d, ok := c.downloaders[hash] + if !ok { + _, err := os.Stat(jsonFilename) + if err != nil { + if !os.IsNotExist(err) { + return nil, nil, err + } + } else { + e := &Entry{} + if err = e.Load(jsonFilename); err != nil { + return nil, nil, err + } + if e.Complete { + f, err := os.Open(dataFilename) + if err != nil { + return nil, nil, err + } + eChan := make(chan *Entry) + go func() { + eChan <- e + close(eChan) + }() + log.Println("[HIT]", rawurl) + return f, eChan, nil + } + } + d = NewDownloader(rawurl, jsonFilename, dataFilename) + go func() { + d.Wait() + c.mutex.Lock() + defer c.mutex.Unlock() + delete(c.downloaders, hash) + c.waitGroup.Done() + }() + c.downloaders[hash] = d + c.waitGroup.Add(1) + } + eChan := make(chan *Entry) + go func() { + eChan <- d.GetEntry() + close(eChan) + }() + log.Println("[MISS]", rawurl) + return NewLiveReader(d, dataFilename), eChan, nil +} + +// TODO: implement some form of "safe abort" for downloads so that the entire +// application doesn't end up spinning its tires waiting for downloads to end + +// Close waits for all downloaders to complete before shutting down. +func (c *Cache) Close() { + c.waitGroup.Wait() +} diff --git a/downloader.go b/downloader.go new file mode 100644 index 0000000..09e6291 --- /dev/null +++ b/downloader.go @@ -0,0 +1,88 @@ +package main + +import ( + "errors" + "io" + "net/http" + "os" + "strconv" + "sync" +) + +// Downloader attempts to download a file from a remote URL. +type Downloader struct { + doneMutex sync.Mutex + err error + entry *Entry + entryMutex sync.Mutex +} + +// NewDownloader creates a new downloader. +func NewDownloader(rawurl, jsonFilename, dataFilename string) *Downloader { + d := &Downloader{} + d.doneMutex.Lock() + d.entryMutex.Lock() + go func() { + defer func() { + d.doneMutex.Unlock() + }() + once := &sync.Once{} + trigger := func() { + once.Do(func() { + d.entryMutex.Unlock() + }) + } + defer trigger() + resp, err := http.Get(rawurl) + if err != nil { + d.err = err + return + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + d.err = errors.New(resp.Status) + return + } + f, err := os.Create(dataFilename) + if err != nil { + d.err = err + return + } + defer f.Close() + d.entry = &Entry{ + URL: rawurl, + ContentLength: strconv.FormatInt(resp.ContentLength, 10), + ContentType: resp.Header.Get("Content-Type"), + LastModified: resp.Header.Get("Last-Modified"), + } + if err = d.entry.Save(jsonFilename); err != nil { + d.err = err + return + } + trigger() + n, err := io.Copy(f, resp.Body) + if err != nil { + d.err = err + return + } + d.entry.ContentLength = strconv.FormatInt(n, 10) + d.entry.Complete = true + d.entry.Save(jsonFilename) + }() + return d +} + +// GetEntry waits until the Entry associated with the download is available. +// This call will block until the entry is available or an error occurs. +func (d *Downloader) GetEntry() *Entry { + d.entryMutex.Lock() + defer d.entryMutex.Unlock() + return d.entry +} + +// Wait will block until the download completes. +func (d *Downloader) Wait() error { + d.doneMutex.Lock() + defer d.doneMutex.Unlock() + return d.err +} diff --git a/entry.go b/entry.go index eb790b9..895e7ec 100644 --- a/entry.go +++ b/entry.go @@ -5,19 +5,22 @@ import ( "os" ) -// Entry represents a file in storage. +// Entry represents an individual item in the cache. type Entry struct { URL string `json:"url"` - ContentLength int64 `json:"content_length"` + Complete bool `json:"complete"` + ContentLength string `json:"content_length"` ContentType string `json:"content_type"` + LastModified string `json:"last_modified"` } -// LoadEntry loads an entry from disk. -func (e *Entry) LoadEntry(filename string) error { +// Load reads the entry from disk. +func (e *Entry) Load(filename string) error { f, err := os.Open(filename) if err != nil { return err } + defer f.Close() return json.NewDecoder(f).Decode(e) } @@ -27,5 +30,6 @@ func (e *Entry) Save(filename string) error { if err != nil { return err } + defer f.Close() return json.NewEncoder(f).Encode(e) } diff --git a/livereader.go b/livereader.go new file mode 100644 index 0000000..e199215 --- /dev/null +++ b/livereader.go @@ -0,0 +1,96 @@ +package main + +import ( + "github.com/fsnotify/fsnotify" + + "io" + "os" +) + +// LiveReader synchronizes with a downloader to read from a file. +type LiveReader struct { + dataFilename string + open chan bool + done chan error + file *os.File + err error + eof bool +} + +// NewLiveReader creates a new live reader. +func NewLiveReader(d *Downloader, dataFilename string) *LiveReader { + l := &LiveReader{ + dataFilename: dataFilename, + open: make(chan bool), + done: make(chan error), + } + go func() { + d.GetEntry() + close(l.open) + l.done <- d.Wait() + close(l.done) + }() + return l +} + +// Read attempts to read data as it is being downloaded. If EOF is reached, +// fsnotify is used to watch for new data being written. The download is not +// complete until the "done" channel receives a value. +func (l *LiveReader) Read(p []byte) (int, error) { + if l.err != nil { + return 0, l.err + } + <-l.open + if l.file == nil { + f, err := os.Open(l.dataFilename) + if err != nil { + return 0, err + } + l.file = f + } + var ( + bytesRead int + watcher *fsnotify.Watcher + ) +loop: + for bytesRead < len(p) { + n, err := l.file.Read(p[bytesRead:]) + bytesRead += n + if err != nil { + if err != io.EOF || l.eof { + l.err = err + break loop + } + if watcher == nil { + watcher, err = fsnotify.NewWatcher() + if err != nil { + l.err = err + break loop + } + defer watcher.Close() + if err = watcher.Add(l.dataFilename); err != nil { + l.err = err + break loop + } + for { + select { + case e := <-watcher.Events: + if e.Op&fsnotify.Write == fsnotify.Write { + continue loop + } + case err = <-l.done: + l.err = err + l.eof = true + } + } + } + } + } + return bytesRead, l.err +} + +// Close frees resources associated with the reader. +func (l *LiveReader) Close() error { + l.file.Close() + return nil +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..98aad28 --- /dev/null +++ b/main.go @@ -0,0 +1,19 @@ +package main + +import ( + "os" + "os/signal" + "syscall" +) + +func main() { + s, err := NewServer(":8000", "/tmp/proxy") + if err != nil { + panic(err) + } + s.Start() + c := make(chan os.Signal) + signal.Notify(c, syscall.SIGINT) + <-c + s.Stop() +} diff --git a/mirrors.json b/mirrors.json deleted file mode 100644 index 59d4385..0000000 --- a/mirrors.json +++ /dev/null @@ -1,396 +0,0 @@ -[ - "http://mirrors.coopvgg.com.ar/ubuntu/", - "http://ubuntu.unc.edu.ar/ubuntu/", - "http://mirrors.asnet.am/ubuntu/", - "http://mirror.aarnet.edu.au/pub/ubuntu/archive/", - "http://ubuntu.uberglobalmirror.com/archive/", - "http://ftp.iinet.net.au/pub/ubuntu/", - "http://mirror.as24220.net/pub/ubuntu-archive/", - "http://mirror.as24220.net/pub/ubuntu/", - "http://mirror.internode.on.net/pub/ubuntu/ubuntu/", - "http://mirror.netspace.net.au/pub/ubuntu/", - "http://mirror.overthewire.com.au/ubuntu/", - "http://ubuntu.mirror.digitalpacific.com.au/archive/", - "http://ubuntu.mirror.serversaustralia.com.au/ubuntu/", - "http://mirror.waia.asn.au/ubuntu/", - "http://ubuntu.inode.at/ubuntu/", - "http://ubuntu.lagis.at/ubuntu/", - "http://ubuntu.uni-klu.ac.at/ubuntu/", - "http://gd.tuwien.ac.at/opsys/linux/ubuntu/archive/", - "http://mirror.datacenter.az/ubuntu/", - "http://mirror.dhakacom.com/ubuntu/", - "http://mirror.dhakacom.com/ubuntu-archive/", - "http://ftp.byfly.by/ubuntu/", - "http://mirror.datacenter.by/ubuntu/", - "http://ftp.belnet.be/ubuntu.com/ubuntu/", - "http://mirror.unix-solutions.be/ubuntu/", - "http://ubuntu-archive.mirror.nucleus.be/", - "http://gaosu.rave.org/ubuntu/", - "http://ubuntu.mirrors.skynet.be/ubuntu/", - "http://ubuntu.mirrors.skynet.be/pub/ubuntu.com/ubuntu/", - "http://archive.ubuntu.com.ba/ubuntu/", - "http://ubuntu.c3sl.ufpr.br/ubuntu/", - "http://mirror.globo.com/ubuntu/archive/", - "http://mirror.unesp.br/ubuntu/", - "http://sft.if.usp.br/ubuntu/", - "http://ubuntu-archive.locaweb.com.br/ubuntu/", - "http://ubuntu.laps.ufpa.br/ubuntu/", - "http://www.las.ic.unicamp.br/pub/ubuntu/", - "http://ubuntu.ipacct.com/ubuntu/", - "http://mirrors.neterra.net/ubuntu/", - "http://mirror.telepoint.bg/ubuntu/", - "http://ubuntu.uni-sofia.bg/ubuntu/", - "http://ubuntu.mirror.iweb.ca/", - "http://archive.ubuntu.mirror.rafal.ca/ubuntu/", - "http://mirror.clibre.uqam.ca/ubuntu/", - "http://mirror.csclub.uwaterloo.ca/ubuntu/", - "http://mirror.it.ubc.ca/ubuntu/", - "http://mirror.its.sfu.ca/mirror/ubuntu/", - "http://ubuntu.bhs.mirrors.ovh.net/ftp.ubuntu.com/ubuntu/", - "http://gpl.savoirfairelinux.net/pub/mirrors/ubuntu/", - "http://mirror.its.dal.ca/ubuntu/", - "http://ubuntu.mirror.rafal.ca/ubuntu/", - "http://mirror.uchile.cl/ubuntu/", - "http://ftp.tecnoera.com/ubuntu/", - "http://mirrors.aliyun.com/ubuntu/", - "http://ubuntu.cn99.com/ubuntu/", - "http://ftp.sjtu.edu.cn/ubuntu/", - "http://mirror.neu.edu.cn/ubuntu/", - "http://mirrors.opencas.org/ubuntu/", - "http://mirrors.sohu.com/ubuntu/", - "http://mirrors.tuna.tsinghua.edu.cn/ubuntu/", - "http://mirrors.ustc.edu.cn/ubuntu/", - "http://mirrors.yun-idc.com/ubuntu/", - "http://mirrors.skyshe.cn/ubuntu/", - "http://mirror.lzu.edu.cn/ubuntu/", - "http://mirrors.cqu.edu.cn/ubuntu/", - "http://mirrors.cug.edu.cn/ubuntu/", - "http://run.hit.edu.cn/ubuntu/", - "http://mirror.edatel.net.co/ubuntu/", - "http://ubuntu.ucr.ac.cr/ubuntu/", - "http://hr.archive.ubuntu.com/ubuntu/", - "http://mirror.vutbr.cz/ubuntu/archive/", - "http://cz.archive.ubuntu.com/ubuntu/", - "http://archive.ubuntu.mirror.dkm.cz/", - "http://ftp.cvut.cz/ubuntu/", - "http://ucho.ignum.cz/ubuntu/", - "http://mirror.easyspeedy.com/ubuntu/", - "http://mirror.one.com/ubuntu/", - "http://ftp.klid.dk/ftp/ubuntu/", - "http://mirrors.dotsrc.org/ubuntu/", - "http://mirrors.telianet.dk/ubuntu/", - "http://mirror.cedia.org.ec/ubuntu/", - "http://ftp.aso.ee/ubuntu/", - "http://mirrors.nic.funet.fi/ubuntu/", - "http://www.nic.funet.fi/pub/mirrors/archive.ubuntu.com/", - "http://ubuntu.trumpetti.atm.tut.fi/ubuntu/", - "http://bouyguestelecom.ubuntu.lafibre.info/ubuntu/", - "http://mirror.plusserver.com/ubuntu/ubuntu/", - "http://ubuntu.mirrors.ovh.net/ftp.ubuntu.com/ubuntu/", - "http://ftp.oleane.net/ubuntu/", - "http://ftp.rezopole.net/ubuntu/", - "http://mirror.ubuntu.ikoula.com/ubuntu/", - "http://mirrors.ircam.fr/pub/ubuntu/archive/", - "http://www-ftp.lip6.fr/pub/linux/distributions/Ubuntu/archive/", - "http://wwwftp.ciril.fr/pub/linux/ubuntu/archives/", - "http://ftp.u-picardie.fr/mirror/ubuntu/ubuntu/", - "http://distrib-coffee.ipsl.jussieu.fr/pub/linux/ubuntu/", - "http://ubuntu.univ-reims.fr/ubuntu/", - "http://pf.archive.ubuntu.com/ubuntu/", - "http://ubuntu.grena.ge/ubuntu/", - "http://ge.archive.ubuntu.com/ubuntu/", - "http://ftp.halifax.rwth-aachen.de/ubuntu/", - "http://ubuntu.mirror.lrz.de/ubuntu/", - "http://ftp.stw-bonn.de/ubuntu/", - "http://ftp.uni-stuttgart.de/ubuntu/", - "http://mirror.de.leaseweb.net/ubuntu/", - "http://ubuntu.mirror.tudos.de/ubuntu/", - "http://ftp.fau.de/ubuntu/", - "http://mirror.23media.de/ubuntu/", - "http://ftp-stud.hs-esslingen.de/ubuntu/", - "http://ftp.uni-kassel.de/ubuntu/ubuntu/", - "http://ftp.uni-kl.de/pub/linux/ubuntu/", - "http://mirror.netcologne.de/ubuntu/", - "http://mirror.serverloft.eu/ubuntu/ubuntu/", - "http://artfiles.org/ubuntu.com/", - "http://debian.charite.de/ubuntu/", - "http://ftp.hawo.stw.uni-erlangen.de/ubuntu/", - "http://ftp.rrzn.uni-hannover.de/pub/mirror/linux/ubuntu/", - "http://ftp.rz.tu-bs.de/pub/mirror/ubuntu-packages/", - "http://ftp.tu-chemnitz.de/pub/linux/ubuntu-ports/", - "http://ftp.tu-ilmenau.de/mirror/ubuntu/", - "http://ftp.uni-bayreuth.de/linux/ubuntu/ubuntu/", - "http://ftp.uni-mainz.de/ubuntu/", - "http://ftp5.gwdg.de/pub/linux/debian/ubuntu/", - "http://mirror.stw-aachen.de/ubuntu/", - "http://mirror2.tuxinator.org/ubuntu/", - "http://vesta.informatik.rwth-aachen.de/ftp/pub/Linux/ubuntu/ubuntu/", - "http://suse.uni-leipzig.de/pub/releases.ubuntu.com/ubuntu/", - "http://ubuntu.unitedcolo.de/ubuntu/", - "http://ftp.hosteurope.de/mirror/archive.ubuntu.com/", - "http://ftp.tu-chemnitz.de/pub/linux/ubuntu/", - "http://de.archive.ubuntu.com/ubuntu/", - "http://de2.archive.ubuntu.com/ubuntu/", - "http://ftp.cc.uoc.gr/mirrors/linux/ubuntu/packages/", - "http://ftp.ntua.gr/ubuntu/", - "http://ubuntu.otenet.gr/", - "http://ubuntu.tsl.gr/", - "http://mirror.greennet.gl/ubuntu/", - "http://ftp.cuhk.edu.hk/pub/Linux/ubuntu/", - "http://ubuntu.01link.hk/", - "http://ubuntu.uhost.hk/", - "http://ftp.kfki.hu/linux/ubuntu/", - "http://ftp.freepark.org/ubuntu/", - "http://speglar.simnet.is/ubuntu/", - "http://ubuntu.hysing.is/ubuntu/", - "http://mirror.cse.iitk.ac.in/ubuntu/", - "http://ftp.iitm.ac.in/ubuntu/", - "http://buaya.klas.or.id/ubuntu/", - "http://kambing.ui.ac.id/ubuntu/", - "http://kartolo.sby.datautama.net.id/ubuntu/", - "http://kebo.pens.ac.id/ubuntu/", - "http://mirror.unej.ac.id/ubuntu/", - "http://mirror.kavalinux.com/ubuntu/", - "http://suro.ubaya.ac.id/ubuntu/", - "http://repo.unpatti.ac.id/ubuntu/", - "http://mirror.poliwangi.ac.id/ubuntu/", - "http://mirror.network32.net/ubuntu/", - "http://ubuntu.asis.io/", - "http://ubuntu.parspack.com/ubuntu/", - "http://mirror.iranserver.com/ubuntu/", - "http://mirror.earthlink.iq/ubuntu/", - "http://ftp.heanet.ie/pub/ubuntu/", - "http://mirror.isoc.org.il/pub/ubuntu/", - "http://ubuntu.mirror.garr.it/ubuntu/", - "http://ubuntu.ictvalleumbra.it/ubuntu/", - "http://mirror.crazynetwork.it/ubuntu/archive/", - "http://giano.com.dist.unige.it/ubuntu/", - "http://ftp.jaist.ac.jp/pub/Linux/ubuntu/", - "http://ftp.riken.jp/Linux/ubuntu/", - "http://ftp.tsukuba.wide.ad.jp/Linux/ubuntu/", - "http://mirror.fairway.ne.jp/ubuntu/", - "http://ubuntutym.u-toyama.ac.jp/ubuntu/", - "http://ubuntu-ashisuto.ubuntulinux.jp/ubuntu/", - "http://www.ftp.ne.jp/Linux/packages/ubuntu/archive/", - "http://mirror.neolabs.kz/ubuntu/", - "http://ubuntu-archive.mirror.liquidtelecom.com/ubuntu/", - "http://ubuntu.mirror.ac.ke/ubuntu/", - "http://kr.archive.ubuntu.com/ubuntu/", - "http://ftp.neowiz.com/ubuntu/", - "http://mirror.premi.st/ubuntu/", - "http://ubuntu-arch.linux.edu.lv/ubuntu/", - "http://ubuntu.koyanet.lv/ubuntu/", - "http://mirror.soften.ktu.lt/ubuntu/", - "http://ubuntu-archive.mirror.serveriai.lt/", - "http://ubuntu.mirror.vu.lt/ubuntu/", - "http://ftp.litnet.lt/ubuntu/", - "http://ubuntu.mirror.lhisp.com/ubuntu/", - "http://ubuntu.mirror.root.lu/ubuntu/", - "http://mirror.blizoo.mk/ubuntu/", - "http://mirror.t-home.mk/ubuntu/", - "http://ubuntu.ipserverone.com/ubuntu/", - "http://ubuntu.tuxuri.com/ubuntu/", - "http://mirror.as43289.net/ubuntu/", - "http://ubuntu.ntc.net.np/ubuntu/", - "http://nl.archive.ubuntu.com/ubuntu/", - "http://ftp.snt.utwente.nl/pub/os/linux/ubuntu/", - "http://mirror.1000mbps.com/ubuntu/", - "http://mirror.i3d.net/pub/ubuntu/", - "http://mirror.nforce.com/pub/linux/ubuntu/", - "http://ubuntu.mirror.cambrium.nl/ubuntu/", - "http://ftp.nluug.nl/os/Linux/distr/ubuntu/", - "http://mirror.nl.leaseweb.net/ubuntu/", - "http://mirror.transip.net/ubuntu/ubuntu/", - "http://mirror.amsiohosting.net/archive.ubuntu.com/", - "http://nl3.archive.ubuntu.com/ubuntu/", - "http://ftp.tudelft.nl/archive.ubuntu.com/", - "http://mirrors.nl.eu.kernel.org/ubuntu/", - "http://mirrors.noction.com/ubuntu/archive/", - "http://osmirror.rug.nl/ubuntu/", - "http://archive.ubuntu.nautile.nc/ubuntu/", - "http://ubuntu.lagoon.nc/ubuntu/", - "http://ftp.citylink.co.nz/ubuntu/", - "http://mirror.xnet.co.nz/pub/ubuntu/", - "http://ucmirror.canterbury.ac.nz/ubuntu/", - "http://nz.archive.ubuntu.com/ubuntu/", - "http://archive.mirror.blix.com/ubuntu/", - "http://no.archive.ubuntu.com/ubuntu/", - "http://ftp.uninett.no/ubuntu/", - "http://ubuntu.uib.no/archive/", - "http://mirror.squ.edu.om/ubuntuarchive/", - "http://mirrors.nayatel.com/ubuntu/", - "http://mirror.rise.ph/ubuntu/", - "http://mirror.pregi.net/ubuntu/", - "http://ftp.icm.edu.pl/pub/Linux/ubuntu/", - "http://mirror.onet.pl/pub/mirrors/ubuntu/", - "http://ftp.agh.edu.pl/ubuntu/", - "http://ubuntu.task.gda.pl/ubuntu/", - "http://ftp.vectranet.pl/ubuntu/", - "http://piotrkosoft.net/pub/mirrors/ubuntu/", - "http://ubuntu.dcc.fc.up.pt/", - "http://archive.ubuntumirror.dei.uc.pt/ubuntu/", - "http://mirrors.fe.up.pt/ubuntu/", - "http://ftp.astral.ro/mirrors/ubuntu.com/ubuntu/", - "http://mirrors.pidginhost.com/ubuntu/", - "http://mirrors.ulbsibiu.ro/ubuntu/", - "http://ubuntu.mirrors.linux.ro/archive/", - "http://ftp.gts.lug.ro/ubuntu/", - "http://linux.nsu.ru/ubuntu/", - "http://ftp.mtu.ru/pub/ubuntu/archive/", - "http://mirror.corbina.net/ubuntu/", - "http://mirror.rol.ru/ubuntu/", - "http://mirror.timeweb.ru/ubuntu/", - "http://mirror.yandex.ru/ubuntu/", - "http://linux.psu.ru/ubuntu/", - "http://mirror.logol.ru/ubuntu/", - "http://download.nus.edu.sg/mirror/ubuntu/", - "http://mirror.nus.edu.sg/ubuntu/", - "http://ftp.energotel.sk/pub/linux/ubuntu/archive/", - "http://tux.rainside.sk/ubuntu/", - "http://ftp.arnes.si/pub/mirrors/ubuntu/", - "http://ubuntu.mirror.neology.co.za/ubuntu/", - "http://ftp.leg.uct.ac.za/ubuntu/", - "http://ftp.udc.es/ubuntu/", - "http://ubuntu.cica.es/ubuntu/", - "http://softlibre.unizar.es/ubuntu/archive/", - "http://ubuntu.uc3m.es/ubuntu/", - "http://ftp.caliu.cat/pub/distribucions/ubuntu/archive/", - "http://ubuntu.grn.cat/ubuntu/", - "http://ftp.acc.umu.se/ubuntu/", - "http://ubuntu.mirror.su.se/ubuntu/", - "http://ftp.availo.se/ubuntu/", - "http://ftp.ds.karen.hj.se/ubuntu/", - "http://ftp.lysator.liu.se/ubuntu/", - "http://mirror.zetup.net/ubuntu/", - "http://mirrors.se.eu.kernel.org/ubuntu/", - "http://ubuntu.cybercomhosting.com/ubuntu/", - "http://archive.ubuntu.csg.uzh.ch/ubuntu/", - "http://mirror.switch.ch/ftp/mirror/ubuntu/", - "http://ubuntu.ethz.ch/ubuntu/", - "http://pkg.adfinis-sygroup.ch/ubuntu/", - "http://free.nchc.org.tw/ubuntu/", - "http://debian.linux.org.tw/ubuntu/", - "http://ftp.ubuntu-tw.net/ubuntu/", - "http://ftp.cs.pu.edu.tw/Linux/Ubuntu/ubuntu/", - "http://ftp.nsysu.edu.tw/Ubuntu/ubuntu/", - "http://ftp.stust.edu.tw/pub/Linux/ubuntu/", - "http://ftp.tku.edu.tw/ubuntu/", - "http://ftp.yzu.edu.tw/ubuntu/", - "http://ftp.ntou.edu.tw/ubuntu/", - "http://ubuntu.cs.nctu.edu.tw/ubuntu/", - "http://ubuntu.stu.edu.tw/ubuntu/", - "http://mirror01.idc.hinet.net/ubuntu/", - "http://mirrors.eastera.tj/ubuntu/", - "http://mirror.aptus.co.tz/pub/ubuntuarchive/", - "http://deb-mirror.habari.co.tz/ubuntu/", - "http://mirror.adminbannok.com/ubuntu/", - "http://mirror.kku.ac.th/ubuntu/", - "http://mirror1.ku.ac.th/ubuntu/", - "http://mirrors.psu.ac.th/ubuntu/", - "http://ubuntu-mirror.totbb.net/ubuntu/", - "http://ubuntu.mirror.tn/ubuntu/", - "http://ftp.linux.org.tr/ubuntu/", - "http://mirror.idealhosting.net.tr/ubuntu/", - "http://mirror.ni.net.tr/ubuntu/", - "http://ubuntu.gnu.gen.tr/ubuntu/", - "http://ubuntu.saglayici.com/ubuntu/", - "http://ubuntu.volia.net/ubuntu-archive/", - "http://ubuntu.ip-connect.vn.ua/", - "http://ubuntu-mirror.neocom.org.ua/ubuntu/", - "http://ubuntu-mirror.telesys.org.ua/ubuntu/", - "http://mirror.mirohost.net/ubuntu/", - "http://ubuntu.org.ua/ubuntu/", - "http://mirror.vorboss.net/ubuntu-archive/", - "http://www.mirrorservice.org/sites/archive.ubuntu.com/ubuntu/", - "http://ftp.ticklers.org/archive.ubuntu.org/ubuntu/", - "http://mirror.as29550.net/archive.ubuntu.com/", - "http://mirror.bytemark.co.uk/ubuntu/", - "http://mirror.ox.ac.uk/sites/archive.ubuntu.com/ubuntu/", - "http://mirror.sax.uk.as61049.net/ubuntu/", - "http://mirror.sov.uk.goscomb.net/ubuntu/", - "http://mirrors.melbourne.co.uk/ubuntu/", - "http://repo.bigstepcloud.com/ubuntu/", - "http://ubuntu.mirrors.uk2.net/ubuntu/", - "http://archive.ubuntu.com/ubuntu/", - "http://ubuntu.positive-internet.com/ubuntu/", - "http://ubuntu.retrosnub.co.uk/ubuntu/", - "http://mirror.math.princeton.edu/pub/ubuntu/", - "http://mirror.n5tech.com/ubuntu/", - "http://mirror.pnl.gov/ubuntu/", - "http://mirror.scalabledns.com/ubuntu/", - "http://mirror.us.leaseweb.net/ubuntu/", - "http://mirrors.rit.edu/ubuntu/", - "http://ubuntu.cs.utah.edu/ubuntu/", - "http://ubuntu.localmsp.org/ubuntu/", - "http://mirror.tocici.com/ubuntu/", - "http://mirror.atlantic.net/ubuntu/", - "http://mirror.cc.columbia.edu/pub/linux/ubuntu/archive/", - "http://mirror.lstn.net/ubuntu/", - "http://mirrors.advancedhosters.com/ubuntu/", - "http://mirrors.bloomu.edu/ubuntu/", - "http://mirrors.us.kernel.org/ubuntu/", - "http://ubuntu.mirror.constant.com/", - "http://ubuntu.osuosl.org/ubuntu/", - "http://ftp.usf.edu/pub/ubuntu/", - "http://lug.mtu.edu/ubuntu/", - "http://mirror.cc.vt.edu/pub2/ubuntu/", - "http://mirror.clarkson.edu/ubuntu/", - "http://mirror.cogentco.com/pub/linux/ubuntu/", - "http://mirror.cs.pitt.edu/ubuntu/archive/", - "http://mirror.htnshost.com/ubuntu/", - "http://mirror.jmu.edu/pub/ubuntu/", - "http://mirror.metrocast.net/ubuntu/", - "http://mirror.nexcess.net/ubuntu/", - "http://mirror.picosecond.org/ubuntu/", - "http://mirror.steadfast.net/ubuntu/", - "http://mirror.symnds.com/ubuntu/", - "http://mirror.team-cymru.org/ubuntu/", - "http://mirror.umd.edu/ubuntu/", - "http://mirrordenver.fdcservers.net/ubuntu/", - "http://mirrors.accretive-networks.net/ubuntu/", - "http://mirrors.arpnetworks.com/Ubuntu/", - "http://mirrors.cat.pdx.edu/ubuntu/", - "http://mirrors.ccs.neu.edu/ubuntu/", - "http://mirrors.easynews.com/linux/ubuntu/", - "http://mirrors.gigenet.com/ubuntuarchive/", - "http://mirrors.liquidweb.com/ubuntu/", - "http://mirrors.maine.edu/ubuntu/", - "http://mirrors.mit.edu/ubuntu/", - "http://mirrors.namecheap.com/ubuntu/", - "http://mirrors.ocf.berkeley.edu/ubuntu/", - "http://mirrors.sonic.net/ubuntu/", - "http://mirrors.syringanetworks.net/ubuntu-archive/", - "http://mirrors.tripadvisor.com/ubuntu/", - "http://mirrors.usinternet.com/ubuntu/archive/", - "http://mirrors.xmission.com/ubuntu/", - "http://pubmirrors.dal.corespace.com/ubuntu/", - "http://ubuntu.mirrors.tds.net/pub/ubuntu/", - "http://ubuntu.mirror.frontiernet.net/ubuntu/", - "http://ubuntu.mirrors.pair.com/archive/", - "http://mirror.stjschools.org/public/ubuntu-archive/", - "http://ubuntu.wallawalla.edu/ubuntu/", - "http://ubuntu.wikimedia.org/ubuntu/", - "http://ubuntuarchive.mirror.nac.net/", - "http://www.club.cc.cmu.edu/pub/ubuntu/", - "http://www.gtlib.gatech.edu/pub/ubuntu/", - "http://archive.linux.duke.edu/ubuntu/", - "http://cosmos.cites.illinois.edu/pub/ubuntu/", - "http://dist1.800hosting.com/ubuntu/", - "http://ftp.utexas.edu/ubuntu/", - "http://mirror.ancl.hawaii.edu/linux/ubuntu/", - "http://mirror.hmc.edu/ubuntu/", - "http://mirror.math.ucdavis.edu/ubuntu/", - "http://mirrors.acm.jhu.edu/ubuntu/", - "http://ubuntu.lionlike.com/ubuntu/", - "http://ubuntu.mirrors.wvstateu.edu/", - "http://ubuntu.securedservers.com/", - "http://us.archive.ubuntu.com/ubuntu/", - "http://reflector.westga.edu/repos/Ubuntu/archive/", - "http://ftp.ussg.iu.edu/linux/ubuntu/", - "http://ubuntu.snet.uz/ubuntu/", - "http://mirror-fpt-telecom.fpt.net/ubuntu/", - "http://mirror.digistar.vn/ubuntu/", - "http://mirrors.nhanhoa.com/ubuntu/", - "http://opensource.xtdv.net/ubuntu/", - "http://mirror.zol.co.zw/ubuntu/" -] diff --git a/mirrors/ubuntu.json b/mirrors/ubuntu.json deleted file mode 100644 index 59d4385..0000000 --- a/mirrors/ubuntu.json +++ /dev/null @@ -1,396 +0,0 @@ -[ - "http://mirrors.coopvgg.com.ar/ubuntu/", - "http://ubuntu.unc.edu.ar/ubuntu/", - "http://mirrors.asnet.am/ubuntu/", - "http://mirror.aarnet.edu.au/pub/ubuntu/archive/", - "http://ubuntu.uberglobalmirror.com/archive/", - "http://ftp.iinet.net.au/pub/ubuntu/", - "http://mirror.as24220.net/pub/ubuntu-archive/", - "http://mirror.as24220.net/pub/ubuntu/", - "http://mirror.internode.on.net/pub/ubuntu/ubuntu/", - "http://mirror.netspace.net.au/pub/ubuntu/", - "http://mirror.overthewire.com.au/ubuntu/", - "http://ubuntu.mirror.digitalpacific.com.au/archive/", - "http://ubuntu.mirror.serversaustralia.com.au/ubuntu/", - "http://mirror.waia.asn.au/ubuntu/", - "http://ubuntu.inode.at/ubuntu/", - "http://ubuntu.lagis.at/ubuntu/", - "http://ubuntu.uni-klu.ac.at/ubuntu/", - "http://gd.tuwien.ac.at/opsys/linux/ubuntu/archive/", - "http://mirror.datacenter.az/ubuntu/", - "http://mirror.dhakacom.com/ubuntu/", - "http://mirror.dhakacom.com/ubuntu-archive/", - "http://ftp.byfly.by/ubuntu/", - "http://mirror.datacenter.by/ubuntu/", - "http://ftp.belnet.be/ubuntu.com/ubuntu/", - "http://mirror.unix-solutions.be/ubuntu/", - "http://ubuntu-archive.mirror.nucleus.be/", - "http://gaosu.rave.org/ubuntu/", - "http://ubuntu.mirrors.skynet.be/ubuntu/", - "http://ubuntu.mirrors.skynet.be/pub/ubuntu.com/ubuntu/", - "http://archive.ubuntu.com.ba/ubuntu/", - "http://ubuntu.c3sl.ufpr.br/ubuntu/", - "http://mirror.globo.com/ubuntu/archive/", - "http://mirror.unesp.br/ubuntu/", - "http://sft.if.usp.br/ubuntu/", - "http://ubuntu-archive.locaweb.com.br/ubuntu/", - "http://ubuntu.laps.ufpa.br/ubuntu/", - "http://www.las.ic.unicamp.br/pub/ubuntu/", - "http://ubuntu.ipacct.com/ubuntu/", - "http://mirrors.neterra.net/ubuntu/", - "http://mirror.telepoint.bg/ubuntu/", - "http://ubuntu.uni-sofia.bg/ubuntu/", - "http://ubuntu.mirror.iweb.ca/", - "http://archive.ubuntu.mirror.rafal.ca/ubuntu/", - "http://mirror.clibre.uqam.ca/ubuntu/", - "http://mirror.csclub.uwaterloo.ca/ubuntu/", - "http://mirror.it.ubc.ca/ubuntu/", - "http://mirror.its.sfu.ca/mirror/ubuntu/", - "http://ubuntu.bhs.mirrors.ovh.net/ftp.ubuntu.com/ubuntu/", - "http://gpl.savoirfairelinux.net/pub/mirrors/ubuntu/", - "http://mirror.its.dal.ca/ubuntu/", - "http://ubuntu.mirror.rafal.ca/ubuntu/", - "http://mirror.uchile.cl/ubuntu/", - "http://ftp.tecnoera.com/ubuntu/", - "http://mirrors.aliyun.com/ubuntu/", - "http://ubuntu.cn99.com/ubuntu/", - "http://ftp.sjtu.edu.cn/ubuntu/", - "http://mirror.neu.edu.cn/ubuntu/", - "http://mirrors.opencas.org/ubuntu/", - "http://mirrors.sohu.com/ubuntu/", - "http://mirrors.tuna.tsinghua.edu.cn/ubuntu/", - "http://mirrors.ustc.edu.cn/ubuntu/", - "http://mirrors.yun-idc.com/ubuntu/", - "http://mirrors.skyshe.cn/ubuntu/", - "http://mirror.lzu.edu.cn/ubuntu/", - "http://mirrors.cqu.edu.cn/ubuntu/", - "http://mirrors.cug.edu.cn/ubuntu/", - "http://run.hit.edu.cn/ubuntu/", - "http://mirror.edatel.net.co/ubuntu/", - "http://ubuntu.ucr.ac.cr/ubuntu/", - "http://hr.archive.ubuntu.com/ubuntu/", - "http://mirror.vutbr.cz/ubuntu/archive/", - "http://cz.archive.ubuntu.com/ubuntu/", - "http://archive.ubuntu.mirror.dkm.cz/", - "http://ftp.cvut.cz/ubuntu/", - "http://ucho.ignum.cz/ubuntu/", - "http://mirror.easyspeedy.com/ubuntu/", - "http://mirror.one.com/ubuntu/", - "http://ftp.klid.dk/ftp/ubuntu/", - "http://mirrors.dotsrc.org/ubuntu/", - "http://mirrors.telianet.dk/ubuntu/", - "http://mirror.cedia.org.ec/ubuntu/", - "http://ftp.aso.ee/ubuntu/", - "http://mirrors.nic.funet.fi/ubuntu/", - "http://www.nic.funet.fi/pub/mirrors/archive.ubuntu.com/", - "http://ubuntu.trumpetti.atm.tut.fi/ubuntu/", - "http://bouyguestelecom.ubuntu.lafibre.info/ubuntu/", - "http://mirror.plusserver.com/ubuntu/ubuntu/", - "http://ubuntu.mirrors.ovh.net/ftp.ubuntu.com/ubuntu/", - "http://ftp.oleane.net/ubuntu/", - "http://ftp.rezopole.net/ubuntu/", - "http://mirror.ubuntu.ikoula.com/ubuntu/", - "http://mirrors.ircam.fr/pub/ubuntu/archive/", - "http://www-ftp.lip6.fr/pub/linux/distributions/Ubuntu/archive/", - "http://wwwftp.ciril.fr/pub/linux/ubuntu/archives/", - "http://ftp.u-picardie.fr/mirror/ubuntu/ubuntu/", - "http://distrib-coffee.ipsl.jussieu.fr/pub/linux/ubuntu/", - "http://ubuntu.univ-reims.fr/ubuntu/", - "http://pf.archive.ubuntu.com/ubuntu/", - "http://ubuntu.grena.ge/ubuntu/", - "http://ge.archive.ubuntu.com/ubuntu/", - "http://ftp.halifax.rwth-aachen.de/ubuntu/", - "http://ubuntu.mirror.lrz.de/ubuntu/", - "http://ftp.stw-bonn.de/ubuntu/", - "http://ftp.uni-stuttgart.de/ubuntu/", - "http://mirror.de.leaseweb.net/ubuntu/", - "http://ubuntu.mirror.tudos.de/ubuntu/", - "http://ftp.fau.de/ubuntu/", - "http://mirror.23media.de/ubuntu/", - "http://ftp-stud.hs-esslingen.de/ubuntu/", - "http://ftp.uni-kassel.de/ubuntu/ubuntu/", - "http://ftp.uni-kl.de/pub/linux/ubuntu/", - "http://mirror.netcologne.de/ubuntu/", - "http://mirror.serverloft.eu/ubuntu/ubuntu/", - "http://artfiles.org/ubuntu.com/", - "http://debian.charite.de/ubuntu/", - "http://ftp.hawo.stw.uni-erlangen.de/ubuntu/", - "http://ftp.rrzn.uni-hannover.de/pub/mirror/linux/ubuntu/", - "http://ftp.rz.tu-bs.de/pub/mirror/ubuntu-packages/", - "http://ftp.tu-chemnitz.de/pub/linux/ubuntu-ports/", - "http://ftp.tu-ilmenau.de/mirror/ubuntu/", - "http://ftp.uni-bayreuth.de/linux/ubuntu/ubuntu/", - "http://ftp.uni-mainz.de/ubuntu/", - "http://ftp5.gwdg.de/pub/linux/debian/ubuntu/", - "http://mirror.stw-aachen.de/ubuntu/", - "http://mirror2.tuxinator.org/ubuntu/", - "http://vesta.informatik.rwth-aachen.de/ftp/pub/Linux/ubuntu/ubuntu/", - "http://suse.uni-leipzig.de/pub/releases.ubuntu.com/ubuntu/", - "http://ubuntu.unitedcolo.de/ubuntu/", - "http://ftp.hosteurope.de/mirror/archive.ubuntu.com/", - "http://ftp.tu-chemnitz.de/pub/linux/ubuntu/", - "http://de.archive.ubuntu.com/ubuntu/", - "http://de2.archive.ubuntu.com/ubuntu/", - "http://ftp.cc.uoc.gr/mirrors/linux/ubuntu/packages/", - "http://ftp.ntua.gr/ubuntu/", - "http://ubuntu.otenet.gr/", - "http://ubuntu.tsl.gr/", - "http://mirror.greennet.gl/ubuntu/", - "http://ftp.cuhk.edu.hk/pub/Linux/ubuntu/", - "http://ubuntu.01link.hk/", - "http://ubuntu.uhost.hk/", - "http://ftp.kfki.hu/linux/ubuntu/", - "http://ftp.freepark.org/ubuntu/", - "http://speglar.simnet.is/ubuntu/", - "http://ubuntu.hysing.is/ubuntu/", - "http://mirror.cse.iitk.ac.in/ubuntu/", - "http://ftp.iitm.ac.in/ubuntu/", - "http://buaya.klas.or.id/ubuntu/", - "http://kambing.ui.ac.id/ubuntu/", - "http://kartolo.sby.datautama.net.id/ubuntu/", - "http://kebo.pens.ac.id/ubuntu/", - "http://mirror.unej.ac.id/ubuntu/", - "http://mirror.kavalinux.com/ubuntu/", - "http://suro.ubaya.ac.id/ubuntu/", - "http://repo.unpatti.ac.id/ubuntu/", - "http://mirror.poliwangi.ac.id/ubuntu/", - "http://mirror.network32.net/ubuntu/", - "http://ubuntu.asis.io/", - "http://ubuntu.parspack.com/ubuntu/", - "http://mirror.iranserver.com/ubuntu/", - "http://mirror.earthlink.iq/ubuntu/", - "http://ftp.heanet.ie/pub/ubuntu/", - "http://mirror.isoc.org.il/pub/ubuntu/", - "http://ubuntu.mirror.garr.it/ubuntu/", - "http://ubuntu.ictvalleumbra.it/ubuntu/", - "http://mirror.crazynetwork.it/ubuntu/archive/", - "http://giano.com.dist.unige.it/ubuntu/", - "http://ftp.jaist.ac.jp/pub/Linux/ubuntu/", - "http://ftp.riken.jp/Linux/ubuntu/", - "http://ftp.tsukuba.wide.ad.jp/Linux/ubuntu/", - "http://mirror.fairway.ne.jp/ubuntu/", - "http://ubuntutym.u-toyama.ac.jp/ubuntu/", - "http://ubuntu-ashisuto.ubuntulinux.jp/ubuntu/", - "http://www.ftp.ne.jp/Linux/packages/ubuntu/archive/", - "http://mirror.neolabs.kz/ubuntu/", - "http://ubuntu-archive.mirror.liquidtelecom.com/ubuntu/", - "http://ubuntu.mirror.ac.ke/ubuntu/", - "http://kr.archive.ubuntu.com/ubuntu/", - "http://ftp.neowiz.com/ubuntu/", - "http://mirror.premi.st/ubuntu/", - "http://ubuntu-arch.linux.edu.lv/ubuntu/", - "http://ubuntu.koyanet.lv/ubuntu/", - "http://mirror.soften.ktu.lt/ubuntu/", - "http://ubuntu-archive.mirror.serveriai.lt/", - "http://ubuntu.mirror.vu.lt/ubuntu/", - "http://ftp.litnet.lt/ubuntu/", - "http://ubuntu.mirror.lhisp.com/ubuntu/", - "http://ubuntu.mirror.root.lu/ubuntu/", - "http://mirror.blizoo.mk/ubuntu/", - "http://mirror.t-home.mk/ubuntu/", - "http://ubuntu.ipserverone.com/ubuntu/", - "http://ubuntu.tuxuri.com/ubuntu/", - "http://mirror.as43289.net/ubuntu/", - "http://ubuntu.ntc.net.np/ubuntu/", - "http://nl.archive.ubuntu.com/ubuntu/", - "http://ftp.snt.utwente.nl/pub/os/linux/ubuntu/", - "http://mirror.1000mbps.com/ubuntu/", - "http://mirror.i3d.net/pub/ubuntu/", - "http://mirror.nforce.com/pub/linux/ubuntu/", - "http://ubuntu.mirror.cambrium.nl/ubuntu/", - "http://ftp.nluug.nl/os/Linux/distr/ubuntu/", - "http://mirror.nl.leaseweb.net/ubuntu/", - "http://mirror.transip.net/ubuntu/ubuntu/", - "http://mirror.amsiohosting.net/archive.ubuntu.com/", - "http://nl3.archive.ubuntu.com/ubuntu/", - "http://ftp.tudelft.nl/archive.ubuntu.com/", - "http://mirrors.nl.eu.kernel.org/ubuntu/", - "http://mirrors.noction.com/ubuntu/archive/", - "http://osmirror.rug.nl/ubuntu/", - "http://archive.ubuntu.nautile.nc/ubuntu/", - "http://ubuntu.lagoon.nc/ubuntu/", - "http://ftp.citylink.co.nz/ubuntu/", - "http://mirror.xnet.co.nz/pub/ubuntu/", - "http://ucmirror.canterbury.ac.nz/ubuntu/", - "http://nz.archive.ubuntu.com/ubuntu/", - "http://archive.mirror.blix.com/ubuntu/", - "http://no.archive.ubuntu.com/ubuntu/", - "http://ftp.uninett.no/ubuntu/", - "http://ubuntu.uib.no/archive/", - "http://mirror.squ.edu.om/ubuntuarchive/", - "http://mirrors.nayatel.com/ubuntu/", - "http://mirror.rise.ph/ubuntu/", - "http://mirror.pregi.net/ubuntu/", - "http://ftp.icm.edu.pl/pub/Linux/ubuntu/", - "http://mirror.onet.pl/pub/mirrors/ubuntu/", - "http://ftp.agh.edu.pl/ubuntu/", - "http://ubuntu.task.gda.pl/ubuntu/", - "http://ftp.vectranet.pl/ubuntu/", - "http://piotrkosoft.net/pub/mirrors/ubuntu/", - "http://ubuntu.dcc.fc.up.pt/", - "http://archive.ubuntumirror.dei.uc.pt/ubuntu/", - "http://mirrors.fe.up.pt/ubuntu/", - "http://ftp.astral.ro/mirrors/ubuntu.com/ubuntu/", - "http://mirrors.pidginhost.com/ubuntu/", - "http://mirrors.ulbsibiu.ro/ubuntu/", - "http://ubuntu.mirrors.linux.ro/archive/", - "http://ftp.gts.lug.ro/ubuntu/", - "http://linux.nsu.ru/ubuntu/", - "http://ftp.mtu.ru/pub/ubuntu/archive/", - "http://mirror.corbina.net/ubuntu/", - "http://mirror.rol.ru/ubuntu/", - "http://mirror.timeweb.ru/ubuntu/", - "http://mirror.yandex.ru/ubuntu/", - "http://linux.psu.ru/ubuntu/", - "http://mirror.logol.ru/ubuntu/", - "http://download.nus.edu.sg/mirror/ubuntu/", - "http://mirror.nus.edu.sg/ubuntu/", - "http://ftp.energotel.sk/pub/linux/ubuntu/archive/", - "http://tux.rainside.sk/ubuntu/", - "http://ftp.arnes.si/pub/mirrors/ubuntu/", - "http://ubuntu.mirror.neology.co.za/ubuntu/", - "http://ftp.leg.uct.ac.za/ubuntu/", - "http://ftp.udc.es/ubuntu/", - "http://ubuntu.cica.es/ubuntu/", - "http://softlibre.unizar.es/ubuntu/archive/", - "http://ubuntu.uc3m.es/ubuntu/", - "http://ftp.caliu.cat/pub/distribucions/ubuntu/archive/", - "http://ubuntu.grn.cat/ubuntu/", - "http://ftp.acc.umu.se/ubuntu/", - "http://ubuntu.mirror.su.se/ubuntu/", - "http://ftp.availo.se/ubuntu/", - "http://ftp.ds.karen.hj.se/ubuntu/", - "http://ftp.lysator.liu.se/ubuntu/", - "http://mirror.zetup.net/ubuntu/", - "http://mirrors.se.eu.kernel.org/ubuntu/", - "http://ubuntu.cybercomhosting.com/ubuntu/", - "http://archive.ubuntu.csg.uzh.ch/ubuntu/", - "http://mirror.switch.ch/ftp/mirror/ubuntu/", - "http://ubuntu.ethz.ch/ubuntu/", - "http://pkg.adfinis-sygroup.ch/ubuntu/", - "http://free.nchc.org.tw/ubuntu/", - "http://debian.linux.org.tw/ubuntu/", - "http://ftp.ubuntu-tw.net/ubuntu/", - "http://ftp.cs.pu.edu.tw/Linux/Ubuntu/ubuntu/", - "http://ftp.nsysu.edu.tw/Ubuntu/ubuntu/", - "http://ftp.stust.edu.tw/pub/Linux/ubuntu/", - "http://ftp.tku.edu.tw/ubuntu/", - "http://ftp.yzu.edu.tw/ubuntu/", - "http://ftp.ntou.edu.tw/ubuntu/", - "http://ubuntu.cs.nctu.edu.tw/ubuntu/", - "http://ubuntu.stu.edu.tw/ubuntu/", - "http://mirror01.idc.hinet.net/ubuntu/", - "http://mirrors.eastera.tj/ubuntu/", - "http://mirror.aptus.co.tz/pub/ubuntuarchive/", - "http://deb-mirror.habari.co.tz/ubuntu/", - "http://mirror.adminbannok.com/ubuntu/", - "http://mirror.kku.ac.th/ubuntu/", - "http://mirror1.ku.ac.th/ubuntu/", - "http://mirrors.psu.ac.th/ubuntu/", - "http://ubuntu-mirror.totbb.net/ubuntu/", - "http://ubuntu.mirror.tn/ubuntu/", - "http://ftp.linux.org.tr/ubuntu/", - "http://mirror.idealhosting.net.tr/ubuntu/", - "http://mirror.ni.net.tr/ubuntu/", - "http://ubuntu.gnu.gen.tr/ubuntu/", - "http://ubuntu.saglayici.com/ubuntu/", - "http://ubuntu.volia.net/ubuntu-archive/", - "http://ubuntu.ip-connect.vn.ua/", - "http://ubuntu-mirror.neocom.org.ua/ubuntu/", - "http://ubuntu-mirror.telesys.org.ua/ubuntu/", - "http://mirror.mirohost.net/ubuntu/", - "http://ubuntu.org.ua/ubuntu/", - "http://mirror.vorboss.net/ubuntu-archive/", - "http://www.mirrorservice.org/sites/archive.ubuntu.com/ubuntu/", - "http://ftp.ticklers.org/archive.ubuntu.org/ubuntu/", - "http://mirror.as29550.net/archive.ubuntu.com/", - "http://mirror.bytemark.co.uk/ubuntu/", - "http://mirror.ox.ac.uk/sites/archive.ubuntu.com/ubuntu/", - "http://mirror.sax.uk.as61049.net/ubuntu/", - "http://mirror.sov.uk.goscomb.net/ubuntu/", - "http://mirrors.melbourne.co.uk/ubuntu/", - "http://repo.bigstepcloud.com/ubuntu/", - "http://ubuntu.mirrors.uk2.net/ubuntu/", - "http://archive.ubuntu.com/ubuntu/", - "http://ubuntu.positive-internet.com/ubuntu/", - "http://ubuntu.retrosnub.co.uk/ubuntu/", - "http://mirror.math.princeton.edu/pub/ubuntu/", - "http://mirror.n5tech.com/ubuntu/", - "http://mirror.pnl.gov/ubuntu/", - "http://mirror.scalabledns.com/ubuntu/", - "http://mirror.us.leaseweb.net/ubuntu/", - "http://mirrors.rit.edu/ubuntu/", - "http://ubuntu.cs.utah.edu/ubuntu/", - "http://ubuntu.localmsp.org/ubuntu/", - "http://mirror.tocici.com/ubuntu/", - "http://mirror.atlantic.net/ubuntu/", - "http://mirror.cc.columbia.edu/pub/linux/ubuntu/archive/", - "http://mirror.lstn.net/ubuntu/", - "http://mirrors.advancedhosters.com/ubuntu/", - "http://mirrors.bloomu.edu/ubuntu/", - "http://mirrors.us.kernel.org/ubuntu/", - "http://ubuntu.mirror.constant.com/", - "http://ubuntu.osuosl.org/ubuntu/", - "http://ftp.usf.edu/pub/ubuntu/", - "http://lug.mtu.edu/ubuntu/", - "http://mirror.cc.vt.edu/pub2/ubuntu/", - "http://mirror.clarkson.edu/ubuntu/", - "http://mirror.cogentco.com/pub/linux/ubuntu/", - "http://mirror.cs.pitt.edu/ubuntu/archive/", - "http://mirror.htnshost.com/ubuntu/", - "http://mirror.jmu.edu/pub/ubuntu/", - "http://mirror.metrocast.net/ubuntu/", - "http://mirror.nexcess.net/ubuntu/", - "http://mirror.picosecond.org/ubuntu/", - "http://mirror.steadfast.net/ubuntu/", - "http://mirror.symnds.com/ubuntu/", - "http://mirror.team-cymru.org/ubuntu/", - "http://mirror.umd.edu/ubuntu/", - "http://mirrordenver.fdcservers.net/ubuntu/", - "http://mirrors.accretive-networks.net/ubuntu/", - "http://mirrors.arpnetworks.com/Ubuntu/", - "http://mirrors.cat.pdx.edu/ubuntu/", - "http://mirrors.ccs.neu.edu/ubuntu/", - "http://mirrors.easynews.com/linux/ubuntu/", - "http://mirrors.gigenet.com/ubuntuarchive/", - "http://mirrors.liquidweb.com/ubuntu/", - "http://mirrors.maine.edu/ubuntu/", - "http://mirrors.mit.edu/ubuntu/", - "http://mirrors.namecheap.com/ubuntu/", - "http://mirrors.ocf.berkeley.edu/ubuntu/", - "http://mirrors.sonic.net/ubuntu/", - "http://mirrors.syringanetworks.net/ubuntu-archive/", - "http://mirrors.tripadvisor.com/ubuntu/", - "http://mirrors.usinternet.com/ubuntu/archive/", - "http://mirrors.xmission.com/ubuntu/", - "http://pubmirrors.dal.corespace.com/ubuntu/", - "http://ubuntu.mirrors.tds.net/pub/ubuntu/", - "http://ubuntu.mirror.frontiernet.net/ubuntu/", - "http://ubuntu.mirrors.pair.com/archive/", - "http://mirror.stjschools.org/public/ubuntu-archive/", - "http://ubuntu.wallawalla.edu/ubuntu/", - "http://ubuntu.wikimedia.org/ubuntu/", - "http://ubuntuarchive.mirror.nac.net/", - "http://www.club.cc.cmu.edu/pub/ubuntu/", - "http://www.gtlib.gatech.edu/pub/ubuntu/", - "http://archive.linux.duke.edu/ubuntu/", - "http://cosmos.cites.illinois.edu/pub/ubuntu/", - "http://dist1.800hosting.com/ubuntu/", - "http://ftp.utexas.edu/ubuntu/", - "http://mirror.ancl.hawaii.edu/linux/ubuntu/", - "http://mirror.hmc.edu/ubuntu/", - "http://mirror.math.ucdavis.edu/ubuntu/", - "http://mirrors.acm.jhu.edu/ubuntu/", - "http://ubuntu.lionlike.com/ubuntu/", - "http://ubuntu.mirrors.wvstateu.edu/", - "http://ubuntu.securedservers.com/", - "http://us.archive.ubuntu.com/ubuntu/", - "http://reflector.westga.edu/repos/Ubuntu/archive/", - "http://ftp.ussg.iu.edu/linux/ubuntu/", - "http://ubuntu.snet.uz/ubuntu/", - "http://mirror-fpt-telecom.fpt.net/ubuntu/", - "http://mirror.digistar.vn/ubuntu/", - "http://mirrors.nhanhoa.com/ubuntu/", - "http://opensource.xtdv.net/ubuntu/", - "http://mirror.zol.co.zw/ubuntu/" -] diff --git a/reader.go b/reader.go deleted file mode 100644 index 92b7272..0000000 --- a/reader.go +++ /dev/null @@ -1,126 +0,0 @@ -package main - -import ( - "github.com/fsnotify/fsnotify" - - "errors" - "io" - "os" - "time" -) - -type Reader struct { - jsonFilename string - dataFilename string - writer *Writer - status Status - statusChanged chan Status - file *os.File -} - -// NewReader reads cache entries from disk. If a writer is supplied, reads are -// synchronized to correspond with data becoming available. -func NewReader(writer *Writer, jsonFilename, dataFilename string) *Reader { - r := &Reader{ - jsonFilename: jsonFilename, - dataFilename: dataFilename, - writer: writer, - status: StatusNone, - statusChanged: make(chan Status), - } - if r.writer != nil { - r.writer.Subscribe(r.statusChanged) - } - return r -} - -// Retrieve the entry description from disk. -func (r *Reader) GetEntry() (*Entry, error) { - if r.writer != nil { - t := time.NewTimer(30 * time.Second) - defer t.Stop() - loop: - for { - select { - case r.status = <-r.statusChanged: - if r.status != StatusNone { - break loop - } - case <-t.C: - return nil, errors.New("timeout exceeded") - } - } - if r.status == StatusError { - return nil, errors.New("writer returned error") - } - } - e := &Entry{} - if err := e.LoadEntry(r.jsonFilename); err != nil { - return nil, err - } - return e, nil -} - -// -func (r *Reader) Open() (err error) { - r.file, err = os.Open(r.dataFilename) - return -} - -// Read attempts to read from the file. For live reads, reads continue until -// the buffer is full or the writer status changes. fsnotify is used to keep -// track of new data being written to the file. -func (r *Reader) Read(p []byte) (n int, err error) { - switch r.status { - case StatusError: - err = errors.New("writer error") - return - case StatusDone: - err = io.EOF - return - default: - for n < len(p) { - var bytesRead int - bytesRead, err = r.file.Read(p[n:]) - n += bytesRead - if err != nil { - if err == io.EOF && r.writer != nil { - err = nil - var watcher *fsnotify.Watcher - watcher, err = fsnotify.NewWatcher() - if err != nil { - return - } - defer watcher.Close() - if err = watcher.Add(r.dataFilename); err != nil { - return - } - loop: - for { - select { - case r.status = <-r.statusChanged: - return - case event := <-watcher.Events: - if event.Op&fsnotify.Write == fsnotify.Write { - break loop - } - } - } - continue - } - return - } - } - return - } -} - -// Close cleans up any open resources. -func (r *Reader) Close() { - if r.file != nil { - r.file.Close() - } - if r.writer != nil { - r.writer.Unsubscribe(r.statusChanged) - } -} diff --git a/server.go b/server.go new file mode 100644 index 0000000..7af780c --- /dev/null +++ b/server.go @@ -0,0 +1,98 @@ +package main + +import ( + "github.com/hectane/go-asyncserver" + + "io" + "log" + "net/http" + "net/url" + "strconv" + "strings" +) + +// Server acts as an HTTP proxy, returning entries from the cache whenever +// possible. Recognized archive mirrors are silently rewritten to avoid +// needless duplication. +type Server struct { + server *server.AsyncServer + cache *Cache +} + +func rewrite(rawurl string) string { + u, err := url.Parse(rawurl) + if err != nil { + return rawurl + } + if strings.HasSuffix(u.Host, ".archive.ubuntu.com") { + u.Host = "archive.ubuntu.com" + u.Path = "/ubuntu/" + rawurl = u.String() + } + return rawurl +} + +func (s *Server) writeHeaders(w http.ResponseWriter, e *Entry) { + if e.ContentType != "" { + w.Header().Set("Content-Type", e.ContentType) + } else { + w.Header().Set("Content-Type", "application/octet-stream") + } + l, err := strconv.ParseInt(e.ContentLength, 10, 64) + if err == nil && l >= 0 { + w.Header().Set("Content-Length", e.ContentLength) + } + if e.LastModified != "" { + w.Header().Set("Last-Modified", e.LastModified) + } + w.WriteHeader(http.StatusOK) +} + +// TODO: support for HEAD requests +// TODO: find a reasonable way for getting errors from eChan + +// ServeHTTP processes an incoming request to the proxy. GET requests are +// served with the storage backend and every other request is (out of +// necessity) rejected since it can't be cached. +func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) { + if req.Method == http.MethodGet { + r, eChan, err := s.cache.GetReader(req.RequestURI) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer r.Close() + e := <-eChan + if e == nil { + http.Error(w, "header retrieval error", http.StatusInternalServerError) + return + } + s.writeHeaders(w, e) + _, err = io.Copy(w, r) + if err != nil { + log.Println("[ERR]", err) + } + } else { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + } +} + +// NewServer creates a new server. +func NewServer(addr, directory string) (*Server, error) { + s := &Server{ + server: server.New(addr), + cache: NewCache(directory), + } + s.server.Handler = s + return s, nil +} + +// Start initializes the server and begins listening for requests. +func (s *Server) Start() error { + return s.server.Start() +} + +// Stop shuts down the server. +func (s *Server) Stop() { + s.server.Stop() +} diff --git a/storage.go b/storage.go deleted file mode 100644 index 6e7fd83..0000000 --- a/storage.go +++ /dev/null @@ -1,57 +0,0 @@ -package main - -import ( - "crypto/md5" - "fmt" - "os" - "path" - "sync" -) - -// Storage provides read and write access to items in the cache. In order to -// avoid race conditions, adding and testing for entries in the cache are -// protected by a mutex. -type Storage struct { - directory string - writers map[string]*Writer - writerDone chan *Writer - mutex sync.Mutex -} - -// NewStorage creates a new storage manager. -func NewStorage(directory string) *Storage { - return &Storage{ - directory: directory, - writers: make(map[string]*Writer), - writerDone: make(chan *Writer), - } -} - -// GetReader returns a *Reader for the specified URL. If the file does not -// exist, both a writer (for downloading the file) and a reader are created. -func (s *Storage) GetReader(url string) (*Reader, error) { - var ( - hash = string(md5.Sum([]byte(url))) - jsonFilename = path.Join(s.directory, fmt.Sprintf("%s.json", hash)) - dataFilename = path.Join(s.directory, fmt.Sprintf("%s.data", hash)) - ) - s.mutex.Lock() - defer s.mutex.Unlock() - w, ok := s.writers[hash] - if ok { - return NewReader(w, jsonFilename, dataFilename), nil - } else { - _, err := os.Stat(jsonFilename) - if err != nil { - if os.IsNotExist(err) { - w = NewWriter(url, jsonFilename, dataFilename, s.writerDone) - s.writers[hash] = w - return NewReader(w, jsonFilename, dataFilename), nil - } else { - return nil, err - } - } else { - return NewReader(nil, jsonFilename, dataFilename), nil - } - } -} diff --git a/writer.go b/writer.go deleted file mode 100644 index a683789..0000000 --- a/writer.go +++ /dev/null @@ -1,103 +0,0 @@ -package main - -import ( - "container/list" - "io" - "net/http" - "os" - "sync" -) - -// Status of download in progress. -type Status int - -const ( - StatusNone = iota // Nothing has happened yet - StatusDownloading // JSON and data files created - StatusError // Error retrieving the data - StatusDone // All data has been retrieved -) - -// Writer connects to a remote URL via HTTP and retrieves the data, writing it -// directly to disk and notifying subscribed readers of its status. -type Writer struct { - mutex sync.Mutex - channels *list.List - status Status -} - -func (w *Writer) sendStatus(statusChan chan<- Status, status Status) { - statusChan <- status -} - -func (w *Writer) setStatus(status Status) { - w.mutex.Lock() - defer w.mutex.Unlock() - w.status = status - for e := w.channels.Front(); e != nil; e = e.Next() { - go w.sendStatus(e.Value.(chan<- Status), status) - } -} - -// NewWriter creates a new writer for the given URL. Status information is -// passed along to subscribed channels. The done channel is used to notify the -// storage system that the download is complete (either success or an error). -func NewWriter(url, jsonFilename, dataFilename string, done chan<- *Writer) *Writer { - w := &Writer{ - channels: list.New(), - } - go func() { - r, err := http.Get(url) - if err != nil { - goto error - } - defer r.Body.Close() - e := &Entry{ - URL: url, - ContentLength: r.ContentLength, - ContentType: r.Header.Get("Content-Type"), - } - if err = e.Save(jsonFilename); err != nil { - goto error - } - f, err := os.Create(dataFilename) - if err != nil { - goto error - } - defer f.Close() - w.setStatus(StatusDownloading) - _, err = io.Copy(f, r.Body) - if err != nil { - goto error - } - w.setStatus(StatusDone) - done <- w - error: - w.setStatus(StatusError) - done <- w - }() - return w -} - -// Subscribe adds a channel to the list to be notified when the writer's status -// changes. The channel will also immediately receive the current status. -func (w *Writer) Subscribe(statusChan chan<- Status) { - w.mutex.Lock() - defer w.mutex.Unlock() - w.channels.PushBack(statusChan) - - // TODO: is "go" necessary here? - go w.sendStatus(statusChan, w.status) -} - -// Unsubscribe removes a channel from the list to be notified. This may occur -// when a client cancels a request, for example. -func (w *Writer) Unsubscribe(statusChan chan<- Status) { - w.mutex.Lock() - defer w.mutex.Unlock() - for e := w.channels.Front(); e != nil; e = e.Next() { - if e.Value.(chan<- Status) == statusChan { - w.channels.Remove(e) - } - } -}