ETag caching

HMCLCore/src/main/java/org/jackhuang/hmcl/task/FileDownloadTask.java

@@ -179,8 +179,10 @@ public class FileDownloadTask extends Task {
     public void execute() throws Exception {
         URL currentURL = url;
 
+        boolean checkETag;
         // Check cache
         if (integrityCheck != null && caching) {
+            checkETag = false;
             Optional<Path> cache = repository.checkExistentFile(candidate, integrityCheck.getAlgorithm(), integrityCheck.getChecksum());
             if (cache.isPresent()) {
                 try {
@@ -191,6 +193,8 @@ public class FileDownloadTask extends Task {
                     Logging.LOG.log(Level.WARNING, "Failed to copy cache files", e);
                 }
             }
+        } else {
+            checkETag = true;
         }
 
         Logging.LOG.log(Level.FINER, "Downloading " + currentURL + " to " + file);
@@ -213,10 +217,16 @@ public class FileDownloadTask extends Task {
                 updateProgress(0);
 
                 HttpURLConnection con = NetworkUtils.createConnection(url);
+                if (checkETag) repository.injectConnection(con);
                 con.connect();
 
-                if (con.getResponseCode() / 100 != 2)
+                if (con.getResponseCode() == 304) {
+                    // Handle cache
+                    Path cache = repository.getCachedRemoteFile(con);
+                    FileUtils.copyFile(cache.toFile(), file);
+                } else if (con.getResponseCode() / 100 != 2) {
                     throw new IOException("Server error, response code: " + con.getResponseCode());
+                }
 
                 int contentLength = con.getContentLength();
                 if (contentLength < 1)
@@ -289,6 +299,21 @@ public class FileDownloadTask extends Task {
                     integrityCheck.performCheck(digest);
                 }
 
+                if (caching) {
+                    try {
+                        if (integrityCheck == null)
+                            repository.cacheFile(file.toPath(), CacheRepository.SHA1, Hex.encodeHex(DigestUtils.digest(CacheRepository.SHA1, file.toPath())));
+                        else
+                            repository.cacheFile(file.toPath(), integrityCheck.getAlgorithm(), integrityCheck.getChecksum());
+                    } catch (IOException e) {
+                        Logging.LOG.log(Level.WARNING, "Failed to cache file", e);
+                    }
+                }
+
+                if (checkETag) {
+                    repository.cacheRemoteFile(file.toPath(), con);
+                }
+
                 return;
             } catch (IOException | IllegalStateException e) {
                 if (temp != null)
@@ -301,17 +326,6 @@ public class FileDownloadTask extends Task {
 
         if (exception != null)
             throw new IOException("Unable to download file " + currentURL + ". " + exception.getMessage(), exception);
-
-        if (caching) {
-            try {
-                if (integrityCheck == null)
-                    repository.cacheFile(file.toPath(), CacheRepository.SHA1, Hex.encodeHex(DigestUtils.digest(CacheRepository.SHA1, file.toPath())));
-                else
-                    repository.cacheFile(file.toPath(), integrityCheck.getAlgorithm(), integrityCheck.getChecksum());
-            } catch (IOException e) {
-                Logging.LOG.log(Level.WARNING, "Failed to cache file", e);
-            }
-        }
     }
 
 }

HMCLCore/src/main/java/org/jackhuang/hmcl/task/GetTask.java

@@ -17,7 +17,9 @@
  */
 package org.jackhuang.hmcl.task;
 
+import org.jackhuang.hmcl.util.CacheRepository;
 import org.jackhuang.hmcl.util.Logging;
+import org.jackhuang.hmcl.util.io.FileUtils;
 import org.jackhuang.hmcl.util.io.IOUtils;
 import org.jackhuang.hmcl.util.io.NetworkUtils;
 
@@ -27,6 +29,8 @@ import java.io.InputStream;
 import java.net.HttpURLConnection;
 import java.net.URL;
 import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.logging.Level;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
@@ -41,6 +45,7 @@ public final class GetTask extends TaskResult<String> {
     private final Charset charset;
     private final int retry;
     private final String id;
+    private CacheRepository repository = CacheRepository.getInstance();
 
     public GetTask(URL url) {
         this(url, ID);
@@ -73,15 +78,33 @@ public final class GetTask extends TaskResult<String> {
         return id;
     }
 
+    public GetTask setCacheRepository(CacheRepository repository) {
+        this.repository = repository;
+        return this;
+    }
+
     @Override
     public void execute() throws Exception {
         Exception exception = null;
+        boolean checkETag = true;
         for (int time = 0; time < retry; ++time) {
             if (time > 0)
                 Logging.LOG.log(Level.WARNING, "Failed to download, repeat times: " + time);
             try {
                 updateProgress(0);
                 HttpURLConnection conn = NetworkUtils.createConnection(url);
+                if (checkETag) repository.injectConnection(conn);
+                conn.connect();
+
+                if (conn.getResponseCode() == 304) {
+                    // Handle cache
+                    Path cache = repository.getCachedRemoteFile(conn);
+                    setResult(FileUtils.readText(cache));
+                    return;
+                } else if (conn.getResponseCode() / 100 != 2) {
+                    throw new IOException("Server error, response code: " + conn.getResponseCode());
+                }
+
                 InputStream input = conn.getInputStream();
                 ByteArrayOutputStream baos = new ByteArrayOutputStream();
                 byte[] buf = new byte[IOUtils.DEFAULT_BUFFER_SIZE];
@@ -100,7 +123,12 @@ public final class GetTask extends TaskResult<String> {
                 if (size > 0 && size != read)
                     throw new IllegalStateException("Not completed! Readed: " + read + ", total size: " + size);
 
-                setResult(baos.toString(charset.name()));
+                String result = baos.toString(charset.name());
+                setResult(result);
+
+                if (checkETag) {
+                    repository.cacheText(result, conn);
+                }
                 return;
             } catch (IOException ex) {
                 exception = ex;

HMCLCore/src/main/java/org/jackhuang/hmcl/util/CacheRepository.java

@@ -17,21 +17,63 @@
  */
 package org.jackhuang.hmcl.util;
 
+import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.OutputStream;
+import java.io.RandomAccessFile;
+import java.net.URLConnection;
+import java.nio.channels.Channels;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.Optional;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.logging.Level;
+import java.util.stream.Stream;
 
+import com.google.gson.JsonParseException;
+import com.google.gson.annotations.SerializedName;
 import org.jackhuang.hmcl.util.function.ExceptionalSupplier;
+import org.jackhuang.hmcl.util.gson.JsonUtils;
 import org.jackhuang.hmcl.util.io.FileUtils;
+import org.jackhuang.hmcl.util.io.IOUtils;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
 
 public class CacheRepository {
     private Path commonDirectory;
     private Path cacheDirectory;
+    private Path indexFile;
+    private Map<String, ETagItem> index;
+    private final ReadWriteLock lock = new ReentrantReadWriteLock();
 
     public void changeDirectory(Path commonDir) {
         commonDirectory = commonDir;
         cacheDirectory = commonDir.resolve("cache");
+        indexFile = cacheDirectory.resolve("etag.json");
+
+        lock.writeLock().lock();
+        try {
+            if (Files.isRegularFile(indexFile)) {
+                ETagIndex raw = JsonUtils.GSON.fromJson(FileUtils.readText(indexFile.toFile()), ETagIndex.class);
+                if (raw == null)
+                    index = new HashMap<>();
+                else
+                    index = joinETagIndexes(raw.eTag);
+            } else
+                index = new HashMap<>();
+        } catch (IOException | JsonParseException e) {
+            Logging.LOG.log(Level.WARNING, "Unable to read index file", e);
+            index = new HashMap<>();
+        } finally {
+            lock.writeLock().unlock();
+        }
     }
 
     public Path getCommonDirectory() {
@@ -100,6 +142,181 @@ public class CacheRepository {
         return cache;
     }
 
+    public Path getCachedRemoteFile(URLConnection conn) throws IOException {
+        String url = conn.getURL().toString();
+        lock.readLock().lock();
+        ETagItem eTagItem;
+        try {
+            eTagItem = index.get(url);
+        } finally {
+            lock.readLock().unlock();
+        }
+        if (eTagItem == null) throw new IOException("Cannot find the URL");
+        if (StringUtils.isBlank(eTagItem.hash) || !fileExists(SHA1, eTagItem.hash)) throw new FileNotFoundException();
+        Path file = getFile(SHA1, eTagItem.hash);
+        if (Files.getLastModifiedTime(file).toMillis() != eTagItem.localLastModified) {
+            String hash = Hex.encodeHex(DigestUtils.digest(SHA1, file));
+            if (!Objects.equals(hash, eTagItem.hash))
+                throw new IOException("This file is modified");
+        }
+        return file;
+    }
+
+    public void injectConnection(URLConnection conn) {
+        String url = conn.getURL().toString();
+        lock.readLock().lock();
+        ETagItem eTagItem;
+        try {
+            eTagItem = index.get(url);
+        } finally {
+            lock.readLock().unlock();
+        }
+        if (eTagItem == null) return;
+        if (eTagItem.eTag != null)
+            conn.setRequestProperty("If-None-Match", eTagItem.eTag);
+        // if (eTagItem.getRemoteLastModified() != null)
+        //     conn.setRequestProperty("If-Modified-Since", eTagItem.getRemoteLastModified());
+    }
+
+    public synchronized void cacheRemoteFile(Path downloaded, URLConnection conn) throws IOException {
+        String eTag = conn.getHeaderField("ETag");
+        if (eTag == null) return;
+        String url = conn.getURL().toString();
+        String lastModified = conn.getHeaderField("Last-Modified");
+        String hash = Hex.encodeHex(DigestUtils.digest(SHA1, downloaded));
+        Path cached = cacheFile(downloaded, SHA1, hash);
+        ETagItem eTagItem = new ETagItem(url, eTag, hash, Files.getLastModifiedTime(cached).toMillis(), lastModified);
+        Lock writeLock = lock.writeLock();
+        writeLock.lock();
+        try {
+            index.put(url, eTagItem);
+            saveETagIndex();
+        } finally {
+            writeLock.unlock();
+        }
+    }
+
+    public synchronized void cacheText(String text, URLConnection conn) throws IOException {
+        String eTag = conn.getHeaderField("ETag");
+        if (eTag == null) return;
+        String url = conn.getURL().toString();
+        String lastModified = conn.getHeaderField("Last-Modified");
+        String hash = Hex.encodeHex(DigestUtils.digest(SHA1, text));
+        Path cached = getFile(SHA1, hash);
+        FileUtils.writeText(cached.toFile(), text);
+        ETagItem eTagItem = new ETagItem(url, eTag, hash, Files.getLastModifiedTime(cached).toMillis(), lastModified);
+        Lock writeLock = lock.writeLock();
+        writeLock.lock();
+        try {
+            index.put(url, eTagItem);
+            saveETagIndex();
+        } finally {
+            writeLock.unlock();
+        }
+    }
+
+    @SafeVarargs
+    private final Map<String, ETagItem> joinETagIndexes(Collection<ETagItem>... indexes) {
+        Map<String, ETagItem> eTags = new ConcurrentHashMap<>();
+
+        Stream<ETagItem> stream = Arrays.stream(indexes).filter(Objects::nonNull).map(Collection::stream)
+                .reduce(Stream.empty(), Stream::concat);
+
+        stream.forEach(eTag -> {
+            eTags.compute(eTag.url, (key, oldValue) -> {
+                if (oldValue == null || oldValue.compareTo(eTag) < 0)
+                    return eTag;
+                else
+                    return oldValue;
+            });
+        });
+
+        return eTags;
+    }
+
+    public void saveETagIndex() throws IOException {
+        try (RandomAccessFile file = new RandomAccessFile(indexFile.toFile(), "rw"); FileChannel channel = file.getChannel()) {
+            FileLock lock = channel.lock();
+            try {
+                ETagIndex indexOnDisk = JsonUtils.GSON.fromJson(new String(IOUtils.readFullyWithoutClosing(Channels.newInputStream(channel)), UTF_8), ETagIndex.class);
+                Map<String, ETagItem> newIndex = joinETagIndexes(indexOnDisk == null ? null : indexOnDisk.eTag, index.values());
+                channel.truncate(0);
+                OutputStream os = Channels.newOutputStream(channel);
+                ETagIndex writeTo = new ETagIndex(newIndex.values());
+                IOUtils.write(JsonUtils.GSON.toJson(writeTo).getBytes(UTF_8), os);
+                this.index = newIndex;
+            } finally {
+                lock.release();
+            }
+        }
+    }
+
+    private class ETagIndex {
+        private final Collection<ETagItem> eTag;
+
+        public ETagIndex() {
+            this.eTag = new HashSet<>();
+        }
+
+        public ETagIndex(Collection<ETagItem> eTags) {
+            this.eTag = new HashSet<>(eTags);
+        }
+    }
+
+    private class ETagItem {
+        private final String url;
+        private final String eTag;
+        private final String hash;
+        @SerializedName("local")
+        private final long localLastModified;
+        @SerializedName("remote")
+        private final String remoteLastModified;
+
+        /**
+         * For Gson.
+         */
+        public ETagItem() {
+            this(null, null, null, 0, null);
+        }
+
+        public ETagItem(String url, String eTag, String hash, long localLastModified, String remoteLastModified) {
+            this.url = url;
+            this.eTag = eTag;
+            this.hash = hash;
+            this.localLastModified = localLastModified;
+            this.remoteLastModified = remoteLastModified;
+        }
+
+        public int compareTo(ETagItem other) {
+            if (!url.equals(other.url))
+                throw new IllegalArgumentException();
+
+            ZonedDateTime thisTime = Lang.ignoringException(() -> ZonedDateTime.parse(remoteLastModified, DateTimeFormatter.RFC_1123_DATE_TIME), null);
+            ZonedDateTime otherTime = Lang.ignoringException(() -> ZonedDateTime.parse(other.remoteLastModified, DateTimeFormatter.RFC_1123_DATE_TIME), null);
+            if (thisTime == null && otherTime == null) return 0;
+            else if (thisTime == null) return -1;
+            else if (otherTime == null) return 1;
+            else return thisTime.compareTo(otherTime);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            ETagItem eTagItem = (ETagItem) o;
+            return localLastModified == eTagItem.localLastModified &&
+                    Objects.equals(url, eTagItem.url) &&
+                    Objects.equals(eTag, eTagItem.eTag) &&
+                    Objects.equals(hash, eTagItem.hash) &&
+                    Objects.equals(remoteLastModified, eTagItem.remoteLastModified);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(url, eTag, hash, localLastModified, remoteLastModified);
+        }
+    }
+
     private static CacheRepository instance = new CacheRepository();
 
     public static CacheRepository getInstance() {

HMCLCore/src/main/java/org/jackhuang/hmcl/util/io/IOUtils.java

@@ -17,10 +17,7 @@
  */
 package org.jackhuang.hmcl.util.io;
 
-import java.io.ByteArrayOutputStream;
+import java.io.*;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
 import java.nio.charset.Charset;
 
 /**
@@ -60,6 +57,14 @@ public final class IOUtils {
         return readFully(stream).toString(charset.name());
     }
 
+    public static void write(String text, OutputStream outputStream) throws IOException {
+        write(text.getBytes(), outputStream);
+    }
+
+    public static void write(byte[] bytes, OutputStream outputStream) throws IOException {
+        copyTo(new ByteArrayInputStream(bytes), outputStream);
+    }
+
     public static void copyTo(InputStream src, OutputStream dest) throws IOException {
         copyTo(src, dest, new byte[DEFAULT_BUFFER_SIZE]);
     }