2
0
mirror of https://github.com/HMCL-dev/HMCL.git synced 2025-02-23 17:19:44 +08:00

ETag caching

This commit is contained in:
huanghongxun 2018-10-09 20:16:16 +08:00
parent daf659373f
commit 5323aaed6d
4 changed files with 282 additions and 18 deletions
HMCLCore/src/main/java/org/jackhuang/hmcl

View File

@ -179,8 +179,10 @@ public class FileDownloadTask extends Task {
public void execute() throws Exception {
URL currentURL = url;
boolean checkETag;
// Check cache
if (integrityCheck != null && caching) {
checkETag = false;
Optional<Path> cache = repository.checkExistentFile(candidate, integrityCheck.getAlgorithm(), integrityCheck.getChecksum());
if (cache.isPresent()) {
try {
@ -191,6 +193,8 @@ public class FileDownloadTask extends Task {
Logging.LOG.log(Level.WARNING, "Failed to copy cache files", e);
}
}
} else {
checkETag = true;
}
Logging.LOG.log(Level.FINER, "Downloading " + currentURL + " to " + file);
@ -213,10 +217,16 @@ public class FileDownloadTask extends Task {
updateProgress(0);
HttpURLConnection con = NetworkUtils.createConnection(url);
if (checkETag) repository.injectConnection(con);
con.connect();
if (con.getResponseCode() / 100 != 2)
if (con.getResponseCode() == 304) {
// Handle cache
Path cache = repository.getCachedRemoteFile(con);
FileUtils.copyFile(cache.toFile(), file);
} else if (con.getResponseCode() / 100 != 2) {
throw new IOException("Server error, response code: " + con.getResponseCode());
}
int contentLength = con.getContentLength();
if (contentLength < 1)
@ -289,6 +299,21 @@ public class FileDownloadTask extends Task {
integrityCheck.performCheck(digest);
}
if (caching) {
try {
if (integrityCheck == null)
repository.cacheFile(file.toPath(), CacheRepository.SHA1, Hex.encodeHex(DigestUtils.digest(CacheRepository.SHA1, file.toPath())));
else
repository.cacheFile(file.toPath(), integrityCheck.getAlgorithm(), integrityCheck.getChecksum());
} catch (IOException e) {
Logging.LOG.log(Level.WARNING, "Failed to cache file", e);
}
}
if (checkETag) {
repository.cacheRemoteFile(file.toPath(), con);
}
return;
} catch (IOException | IllegalStateException e) {
if (temp != null)
@ -301,17 +326,6 @@ public class FileDownloadTask extends Task {
if (exception != null)
throw new IOException("Unable to download file " + currentURL + ". " + exception.getMessage(), exception);
if (caching) {
try {
if (integrityCheck == null)
repository.cacheFile(file.toPath(), CacheRepository.SHA1, Hex.encodeHex(DigestUtils.digest(CacheRepository.SHA1, file.toPath())));
else
repository.cacheFile(file.toPath(), integrityCheck.getAlgorithm(), integrityCheck.getChecksum());
} catch (IOException e) {
Logging.LOG.log(Level.WARNING, "Failed to cache file", e);
}
}
}
}

View File

@ -17,7 +17,9 @@
*/
package org.jackhuang.hmcl.task;
import org.jackhuang.hmcl.util.CacheRepository;
import org.jackhuang.hmcl.util.Logging;
import org.jackhuang.hmcl.util.io.FileUtils;
import org.jackhuang.hmcl.util.io.IOUtils;
import org.jackhuang.hmcl.util.io.NetworkUtils;
@ -27,6 +29,8 @@ import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.logging.Level;
import static java.nio.charset.StandardCharsets.UTF_8;
@ -41,6 +45,7 @@ public final class GetTask extends TaskResult<String> {
private final Charset charset;
private final int retry;
private final String id;
private CacheRepository repository = CacheRepository.getInstance();
public GetTask(URL url) {
this(url, ID);
@ -73,15 +78,33 @@ public final class GetTask extends TaskResult<String> {
return id;
}
public GetTask setCacheRepository(CacheRepository repository) {
this.repository = repository;
return this;
}
@Override
public void execute() throws Exception {
Exception exception = null;
boolean checkETag = true;
for (int time = 0; time < retry; ++time) {
if (time > 0)
Logging.LOG.log(Level.WARNING, "Failed to download, repeat times: " + time);
try {
updateProgress(0);
HttpURLConnection conn = NetworkUtils.createConnection(url);
if (checkETag) repository.injectConnection(conn);
conn.connect();
if (conn.getResponseCode() == 304) {
// Handle cache
Path cache = repository.getCachedRemoteFile(conn);
setResult(FileUtils.readText(cache));
return;
} else if (conn.getResponseCode() / 100 != 2) {
throw new IOException("Server error, response code: " + conn.getResponseCode());
}
InputStream input = conn.getInputStream();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
byte[] buf = new byte[IOUtils.DEFAULT_BUFFER_SIZE];
@ -100,7 +123,12 @@ public final class GetTask extends TaskResult<String> {
if (size > 0 && size != read)
throw new IllegalStateException("Not completed! Readed: " + read + ", total size: " + size);
setResult(baos.toString(charset.name()));
String result = baos.toString(charset.name());
setResult(result);
if (checkETag) {
repository.cacheText(result, conn);
}
return;
} catch (IOException ex) {
exception = ex;

View File

@ -17,21 +17,63 @@
*/
package org.jackhuang.hmcl.util;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.net.URLConnection;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Optional;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.logging.Level;
import java.util.stream.Stream;
import com.google.gson.JsonParseException;
import com.google.gson.annotations.SerializedName;
import org.jackhuang.hmcl.util.function.ExceptionalSupplier;
import org.jackhuang.hmcl.util.gson.JsonUtils;
import org.jackhuang.hmcl.util.io.FileUtils;
import org.jackhuang.hmcl.util.io.IOUtils;
import static java.nio.charset.StandardCharsets.UTF_8;
public class CacheRepository {
private Path commonDirectory;
private Path cacheDirectory;
private Path indexFile;
private Map<String, ETagItem> index;
private final ReadWriteLock lock = new ReentrantReadWriteLock();
public void changeDirectory(Path commonDir) {
commonDirectory = commonDir;
cacheDirectory = commonDir.resolve("cache");
indexFile = cacheDirectory.resolve("etag.json");
lock.writeLock().lock();
try {
if (Files.isRegularFile(indexFile)) {
ETagIndex raw = JsonUtils.GSON.fromJson(FileUtils.readText(indexFile.toFile()), ETagIndex.class);
if (raw == null)
index = new HashMap<>();
else
index = joinETagIndexes(raw.eTag);
} else
index = new HashMap<>();
} catch (IOException | JsonParseException e) {
Logging.LOG.log(Level.WARNING, "Unable to read index file", e);
index = new HashMap<>();
} finally {
lock.writeLock().unlock();
}
}
public Path getCommonDirectory() {
@ -100,6 +142,181 @@ public class CacheRepository {
return cache;
}
public Path getCachedRemoteFile(URLConnection conn) throws IOException {
String url = conn.getURL().toString();
lock.readLock().lock();
ETagItem eTagItem;
try {
eTagItem = index.get(url);
} finally {
lock.readLock().unlock();
}
if (eTagItem == null) throw new IOException("Cannot find the URL");
if (StringUtils.isBlank(eTagItem.hash) || !fileExists(SHA1, eTagItem.hash)) throw new FileNotFoundException();
Path file = getFile(SHA1, eTagItem.hash);
if (Files.getLastModifiedTime(file).toMillis() != eTagItem.localLastModified) {
String hash = Hex.encodeHex(DigestUtils.digest(SHA1, file));
if (!Objects.equals(hash, eTagItem.hash))
throw new IOException("This file is modified");
}
return file;
}
public void injectConnection(URLConnection conn) {
String url = conn.getURL().toString();
lock.readLock().lock();
ETagItem eTagItem;
try {
eTagItem = index.get(url);
} finally {
lock.readLock().unlock();
}
if (eTagItem == null) return;
if (eTagItem.eTag != null)
conn.setRequestProperty("If-None-Match", eTagItem.eTag);
// if (eTagItem.getRemoteLastModified() != null)
// conn.setRequestProperty("If-Modified-Since", eTagItem.getRemoteLastModified());
}
public synchronized void cacheRemoteFile(Path downloaded, URLConnection conn) throws IOException {
String eTag = conn.getHeaderField("ETag");
if (eTag == null) return;
String url = conn.getURL().toString();
String lastModified = conn.getHeaderField("Last-Modified");
String hash = Hex.encodeHex(DigestUtils.digest(SHA1, downloaded));
Path cached = cacheFile(downloaded, SHA1, hash);
ETagItem eTagItem = new ETagItem(url, eTag, hash, Files.getLastModifiedTime(cached).toMillis(), lastModified);
Lock writeLock = lock.writeLock();
writeLock.lock();
try {
index.put(url, eTagItem);
saveETagIndex();
} finally {
writeLock.unlock();
}
}
public synchronized void cacheText(String text, URLConnection conn) throws IOException {
String eTag = conn.getHeaderField("ETag");
if (eTag == null) return;
String url = conn.getURL().toString();
String lastModified = conn.getHeaderField("Last-Modified");
String hash = Hex.encodeHex(DigestUtils.digest(SHA1, text));
Path cached = getFile(SHA1, hash);
FileUtils.writeText(cached.toFile(), text);
ETagItem eTagItem = new ETagItem(url, eTag, hash, Files.getLastModifiedTime(cached).toMillis(), lastModified);
Lock writeLock = lock.writeLock();
writeLock.lock();
try {
index.put(url, eTagItem);
saveETagIndex();
} finally {
writeLock.unlock();
}
}
@SafeVarargs
private final Map<String, ETagItem> joinETagIndexes(Collection<ETagItem>... indexes) {
Map<String, ETagItem> eTags = new ConcurrentHashMap<>();
Stream<ETagItem> stream = Arrays.stream(indexes).filter(Objects::nonNull).map(Collection::stream)
.reduce(Stream.empty(), Stream::concat);
stream.forEach(eTag -> {
eTags.compute(eTag.url, (key, oldValue) -> {
if (oldValue == null || oldValue.compareTo(eTag) < 0)
return eTag;
else
return oldValue;
});
});
return eTags;
}
public void saveETagIndex() throws IOException {
try (RandomAccessFile file = new RandomAccessFile(indexFile.toFile(), "rw"); FileChannel channel = file.getChannel()) {
FileLock lock = channel.lock();
try {
ETagIndex indexOnDisk = JsonUtils.GSON.fromJson(new String(IOUtils.readFullyWithoutClosing(Channels.newInputStream(channel)), UTF_8), ETagIndex.class);
Map<String, ETagItem> newIndex = joinETagIndexes(indexOnDisk == null ? null : indexOnDisk.eTag, index.values());
channel.truncate(0);
OutputStream os = Channels.newOutputStream(channel);
ETagIndex writeTo = new ETagIndex(newIndex.values());
IOUtils.write(JsonUtils.GSON.toJson(writeTo).getBytes(UTF_8), os);
this.index = newIndex;
} finally {
lock.release();
}
}
}
private class ETagIndex {
private final Collection<ETagItem> eTag;
public ETagIndex() {
this.eTag = new HashSet<>();
}
public ETagIndex(Collection<ETagItem> eTags) {
this.eTag = new HashSet<>(eTags);
}
}
private class ETagItem {
private final String url;
private final String eTag;
private final String hash;
@SerializedName("local")
private final long localLastModified;
@SerializedName("remote")
private final String remoteLastModified;
/**
* For Gson.
*/
public ETagItem() {
this(null, null, null, 0, null);
}
public ETagItem(String url, String eTag, String hash, long localLastModified, String remoteLastModified) {
this.url = url;
this.eTag = eTag;
this.hash = hash;
this.localLastModified = localLastModified;
this.remoteLastModified = remoteLastModified;
}
public int compareTo(ETagItem other) {
if (!url.equals(other.url))
throw new IllegalArgumentException();
ZonedDateTime thisTime = Lang.ignoringException(() -> ZonedDateTime.parse(remoteLastModified, DateTimeFormatter.RFC_1123_DATE_TIME), null);
ZonedDateTime otherTime = Lang.ignoringException(() -> ZonedDateTime.parse(other.remoteLastModified, DateTimeFormatter.RFC_1123_DATE_TIME), null);
if (thisTime == null && otherTime == null) return 0;
else if (thisTime == null) return -1;
else if (otherTime == null) return 1;
else return thisTime.compareTo(otherTime);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ETagItem eTagItem = (ETagItem) o;
return localLastModified == eTagItem.localLastModified &&
Objects.equals(url, eTagItem.url) &&
Objects.equals(eTag, eTagItem.eTag) &&
Objects.equals(hash, eTagItem.hash) &&
Objects.equals(remoteLastModified, eTagItem.remoteLastModified);
}
@Override
public int hashCode() {
return Objects.hash(url, eTag, hash, localLastModified, remoteLastModified);
}
}
private static CacheRepository instance = new CacheRepository();
public static CacheRepository getInstance() {

View File

@ -17,10 +17,7 @@
*/
package org.jackhuang.hmcl.util.io;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.*;
import java.nio.charset.Charset;
/**
@ -60,6 +57,14 @@ public final class IOUtils {
return readFully(stream).toString(charset.name());
}
public static void write(String text, OutputStream outputStream) throws IOException {
write(text.getBytes(), outputStream);
}
public static void write(byte[] bytes, OutputStream outputStream) throws IOException {
copyTo(new ByteArrayInputStream(bytes), outputStream);
}
public static void copyTo(InputStream src, OutputStream dest) throws IOException {
copyTo(src, dest, new byte[DEFAULT_BUFFER_SIZE]);
}