mirror of
https://github.com/HMCL-dev/HMCL.git
synced 2025-02-23 17:19:44 +08:00
ETag caching
This commit is contained in:
parent
daf659373f
commit
5323aaed6d
HMCLCore/src/main/java/org/jackhuang/hmcl
@ -179,8 +179,10 @@ public class FileDownloadTask extends Task {
|
||||
public void execute() throws Exception {
|
||||
URL currentURL = url;
|
||||
|
||||
boolean checkETag;
|
||||
// Check cache
|
||||
if (integrityCheck != null && caching) {
|
||||
checkETag = false;
|
||||
Optional<Path> cache = repository.checkExistentFile(candidate, integrityCheck.getAlgorithm(), integrityCheck.getChecksum());
|
||||
if (cache.isPresent()) {
|
||||
try {
|
||||
@ -191,6 +193,8 @@ public class FileDownloadTask extends Task {
|
||||
Logging.LOG.log(Level.WARNING, "Failed to copy cache files", e);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
checkETag = true;
|
||||
}
|
||||
|
||||
Logging.LOG.log(Level.FINER, "Downloading " + currentURL + " to " + file);
|
||||
@ -213,10 +217,16 @@ public class FileDownloadTask extends Task {
|
||||
updateProgress(0);
|
||||
|
||||
HttpURLConnection con = NetworkUtils.createConnection(url);
|
||||
if (checkETag) repository.injectConnection(con);
|
||||
con.connect();
|
||||
|
||||
if (con.getResponseCode() / 100 != 2)
|
||||
if (con.getResponseCode() == 304) {
|
||||
// Handle cache
|
||||
Path cache = repository.getCachedRemoteFile(con);
|
||||
FileUtils.copyFile(cache.toFile(), file);
|
||||
} else if (con.getResponseCode() / 100 != 2) {
|
||||
throw new IOException("Server error, response code: " + con.getResponseCode());
|
||||
}
|
||||
|
||||
int contentLength = con.getContentLength();
|
||||
if (contentLength < 1)
|
||||
@ -289,6 +299,21 @@ public class FileDownloadTask extends Task {
|
||||
integrityCheck.performCheck(digest);
|
||||
}
|
||||
|
||||
if (caching) {
|
||||
try {
|
||||
if (integrityCheck == null)
|
||||
repository.cacheFile(file.toPath(), CacheRepository.SHA1, Hex.encodeHex(DigestUtils.digest(CacheRepository.SHA1, file.toPath())));
|
||||
else
|
||||
repository.cacheFile(file.toPath(), integrityCheck.getAlgorithm(), integrityCheck.getChecksum());
|
||||
} catch (IOException e) {
|
||||
Logging.LOG.log(Level.WARNING, "Failed to cache file", e);
|
||||
}
|
||||
}
|
||||
|
||||
if (checkETag) {
|
||||
repository.cacheRemoteFile(file.toPath(), con);
|
||||
}
|
||||
|
||||
return;
|
||||
} catch (IOException | IllegalStateException e) {
|
||||
if (temp != null)
|
||||
@ -301,17 +326,6 @@ public class FileDownloadTask extends Task {
|
||||
|
||||
if (exception != null)
|
||||
throw new IOException("Unable to download file " + currentURL + ". " + exception.getMessage(), exception);
|
||||
|
||||
if (caching) {
|
||||
try {
|
||||
if (integrityCheck == null)
|
||||
repository.cacheFile(file.toPath(), CacheRepository.SHA1, Hex.encodeHex(DigestUtils.digest(CacheRepository.SHA1, file.toPath())));
|
||||
else
|
||||
repository.cacheFile(file.toPath(), integrityCheck.getAlgorithm(), integrityCheck.getChecksum());
|
||||
} catch (IOException e) {
|
||||
Logging.LOG.log(Level.WARNING, "Failed to cache file", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -17,7 +17,9 @@
|
||||
*/
|
||||
package org.jackhuang.hmcl.task;
|
||||
|
||||
import org.jackhuang.hmcl.util.CacheRepository;
|
||||
import org.jackhuang.hmcl.util.Logging;
|
||||
import org.jackhuang.hmcl.util.io.FileUtils;
|
||||
import org.jackhuang.hmcl.util.io.IOUtils;
|
||||
import org.jackhuang.hmcl.util.io.NetworkUtils;
|
||||
|
||||
@ -27,6 +29,8 @@ import java.io.InputStream;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URL;
|
||||
import java.nio.charset.Charset;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.logging.Level;
|
||||
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
@ -41,6 +45,7 @@ public final class GetTask extends TaskResult<String> {
|
||||
private final Charset charset;
|
||||
private final int retry;
|
||||
private final String id;
|
||||
private CacheRepository repository = CacheRepository.getInstance();
|
||||
|
||||
public GetTask(URL url) {
|
||||
this(url, ID);
|
||||
@ -73,15 +78,33 @@ public final class GetTask extends TaskResult<String> {
|
||||
return id;
|
||||
}
|
||||
|
||||
public GetTask setCacheRepository(CacheRepository repository) {
|
||||
this.repository = repository;
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void execute() throws Exception {
|
||||
Exception exception = null;
|
||||
boolean checkETag = true;
|
||||
for (int time = 0; time < retry; ++time) {
|
||||
if (time > 0)
|
||||
Logging.LOG.log(Level.WARNING, "Failed to download, repeat times: " + time);
|
||||
try {
|
||||
updateProgress(0);
|
||||
HttpURLConnection conn = NetworkUtils.createConnection(url);
|
||||
if (checkETag) repository.injectConnection(conn);
|
||||
conn.connect();
|
||||
|
||||
if (conn.getResponseCode() == 304) {
|
||||
// Handle cache
|
||||
Path cache = repository.getCachedRemoteFile(conn);
|
||||
setResult(FileUtils.readText(cache));
|
||||
return;
|
||||
} else if (conn.getResponseCode() / 100 != 2) {
|
||||
throw new IOException("Server error, response code: " + conn.getResponseCode());
|
||||
}
|
||||
|
||||
InputStream input = conn.getInputStream();
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
byte[] buf = new byte[IOUtils.DEFAULT_BUFFER_SIZE];
|
||||
@ -100,7 +123,12 @@ public final class GetTask extends TaskResult<String> {
|
||||
if (size > 0 && size != read)
|
||||
throw new IllegalStateException("Not completed! Readed: " + read + ", total size: " + size);
|
||||
|
||||
setResult(baos.toString(charset.name()));
|
||||
String result = baos.toString(charset.name());
|
||||
setResult(result);
|
||||
|
||||
if (checkETag) {
|
||||
repository.cacheText(result, conn);
|
||||
}
|
||||
return;
|
||||
} catch (IOException ex) {
|
||||
exception = ex;
|
||||
|
@ -17,21 +17,63 @@
|
||||
*/
|
||||
package org.jackhuang.hmcl.util;
|
||||
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.io.RandomAccessFile;
|
||||
import java.net.URLConnection;
|
||||
import java.nio.channels.Channels;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.nio.channels.FileLock;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Optional;
|
||||
import java.time.ZonedDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.locks.Lock;
|
||||
import java.util.concurrent.locks.ReadWriteLock;
|
||||
import java.util.concurrent.locks.ReentrantReadWriteLock;
|
||||
import java.util.logging.Level;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import com.google.gson.JsonParseException;
|
||||
import com.google.gson.annotations.SerializedName;
|
||||
import org.jackhuang.hmcl.util.function.ExceptionalSupplier;
|
||||
import org.jackhuang.hmcl.util.gson.JsonUtils;
|
||||
import org.jackhuang.hmcl.util.io.FileUtils;
|
||||
import org.jackhuang.hmcl.util.io.IOUtils;
|
||||
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
public class CacheRepository {
|
||||
private Path commonDirectory;
|
||||
private Path cacheDirectory;
|
||||
private Path indexFile;
|
||||
private Map<String, ETagItem> index;
|
||||
private final ReadWriteLock lock = new ReentrantReadWriteLock();
|
||||
|
||||
public void changeDirectory(Path commonDir) {
|
||||
commonDirectory = commonDir;
|
||||
cacheDirectory = commonDir.resolve("cache");
|
||||
indexFile = cacheDirectory.resolve("etag.json");
|
||||
|
||||
lock.writeLock().lock();
|
||||
try {
|
||||
if (Files.isRegularFile(indexFile)) {
|
||||
ETagIndex raw = JsonUtils.GSON.fromJson(FileUtils.readText(indexFile.toFile()), ETagIndex.class);
|
||||
if (raw == null)
|
||||
index = new HashMap<>();
|
||||
else
|
||||
index = joinETagIndexes(raw.eTag);
|
||||
} else
|
||||
index = new HashMap<>();
|
||||
} catch (IOException | JsonParseException e) {
|
||||
Logging.LOG.log(Level.WARNING, "Unable to read index file", e);
|
||||
index = new HashMap<>();
|
||||
} finally {
|
||||
lock.writeLock().unlock();
|
||||
}
|
||||
}
|
||||
|
||||
public Path getCommonDirectory() {
|
||||
@ -100,6 +142,181 @@ public class CacheRepository {
|
||||
return cache;
|
||||
}
|
||||
|
||||
public Path getCachedRemoteFile(URLConnection conn) throws IOException {
|
||||
String url = conn.getURL().toString();
|
||||
lock.readLock().lock();
|
||||
ETagItem eTagItem;
|
||||
try {
|
||||
eTagItem = index.get(url);
|
||||
} finally {
|
||||
lock.readLock().unlock();
|
||||
}
|
||||
if (eTagItem == null) throw new IOException("Cannot find the URL");
|
||||
if (StringUtils.isBlank(eTagItem.hash) || !fileExists(SHA1, eTagItem.hash)) throw new FileNotFoundException();
|
||||
Path file = getFile(SHA1, eTagItem.hash);
|
||||
if (Files.getLastModifiedTime(file).toMillis() != eTagItem.localLastModified) {
|
||||
String hash = Hex.encodeHex(DigestUtils.digest(SHA1, file));
|
||||
if (!Objects.equals(hash, eTagItem.hash))
|
||||
throw new IOException("This file is modified");
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
public void injectConnection(URLConnection conn) {
|
||||
String url = conn.getURL().toString();
|
||||
lock.readLock().lock();
|
||||
ETagItem eTagItem;
|
||||
try {
|
||||
eTagItem = index.get(url);
|
||||
} finally {
|
||||
lock.readLock().unlock();
|
||||
}
|
||||
if (eTagItem == null) return;
|
||||
if (eTagItem.eTag != null)
|
||||
conn.setRequestProperty("If-None-Match", eTagItem.eTag);
|
||||
// if (eTagItem.getRemoteLastModified() != null)
|
||||
// conn.setRequestProperty("If-Modified-Since", eTagItem.getRemoteLastModified());
|
||||
}
|
||||
|
||||
public synchronized void cacheRemoteFile(Path downloaded, URLConnection conn) throws IOException {
|
||||
String eTag = conn.getHeaderField("ETag");
|
||||
if (eTag == null) return;
|
||||
String url = conn.getURL().toString();
|
||||
String lastModified = conn.getHeaderField("Last-Modified");
|
||||
String hash = Hex.encodeHex(DigestUtils.digest(SHA1, downloaded));
|
||||
Path cached = cacheFile(downloaded, SHA1, hash);
|
||||
ETagItem eTagItem = new ETagItem(url, eTag, hash, Files.getLastModifiedTime(cached).toMillis(), lastModified);
|
||||
Lock writeLock = lock.writeLock();
|
||||
writeLock.lock();
|
||||
try {
|
||||
index.put(url, eTagItem);
|
||||
saveETagIndex();
|
||||
} finally {
|
||||
writeLock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized void cacheText(String text, URLConnection conn) throws IOException {
|
||||
String eTag = conn.getHeaderField("ETag");
|
||||
if (eTag == null) return;
|
||||
String url = conn.getURL().toString();
|
||||
String lastModified = conn.getHeaderField("Last-Modified");
|
||||
String hash = Hex.encodeHex(DigestUtils.digest(SHA1, text));
|
||||
Path cached = getFile(SHA1, hash);
|
||||
FileUtils.writeText(cached.toFile(), text);
|
||||
ETagItem eTagItem = new ETagItem(url, eTag, hash, Files.getLastModifiedTime(cached).toMillis(), lastModified);
|
||||
Lock writeLock = lock.writeLock();
|
||||
writeLock.lock();
|
||||
try {
|
||||
index.put(url, eTagItem);
|
||||
saveETagIndex();
|
||||
} finally {
|
||||
writeLock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
@SafeVarargs
|
||||
private final Map<String, ETagItem> joinETagIndexes(Collection<ETagItem>... indexes) {
|
||||
Map<String, ETagItem> eTags = new ConcurrentHashMap<>();
|
||||
|
||||
Stream<ETagItem> stream = Arrays.stream(indexes).filter(Objects::nonNull).map(Collection::stream)
|
||||
.reduce(Stream.empty(), Stream::concat);
|
||||
|
||||
stream.forEach(eTag -> {
|
||||
eTags.compute(eTag.url, (key, oldValue) -> {
|
||||
if (oldValue == null || oldValue.compareTo(eTag) < 0)
|
||||
return eTag;
|
||||
else
|
||||
return oldValue;
|
||||
});
|
||||
});
|
||||
|
||||
return eTags;
|
||||
}
|
||||
|
||||
public void saveETagIndex() throws IOException {
|
||||
try (RandomAccessFile file = new RandomAccessFile(indexFile.toFile(), "rw"); FileChannel channel = file.getChannel()) {
|
||||
FileLock lock = channel.lock();
|
||||
try {
|
||||
ETagIndex indexOnDisk = JsonUtils.GSON.fromJson(new String(IOUtils.readFullyWithoutClosing(Channels.newInputStream(channel)), UTF_8), ETagIndex.class);
|
||||
Map<String, ETagItem> newIndex = joinETagIndexes(indexOnDisk == null ? null : indexOnDisk.eTag, index.values());
|
||||
channel.truncate(0);
|
||||
OutputStream os = Channels.newOutputStream(channel);
|
||||
ETagIndex writeTo = new ETagIndex(newIndex.values());
|
||||
IOUtils.write(JsonUtils.GSON.toJson(writeTo).getBytes(UTF_8), os);
|
||||
this.index = newIndex;
|
||||
} finally {
|
||||
lock.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private class ETagIndex {
|
||||
private final Collection<ETagItem> eTag;
|
||||
|
||||
public ETagIndex() {
|
||||
this.eTag = new HashSet<>();
|
||||
}
|
||||
|
||||
public ETagIndex(Collection<ETagItem> eTags) {
|
||||
this.eTag = new HashSet<>(eTags);
|
||||
}
|
||||
}
|
||||
|
||||
private class ETagItem {
|
||||
private final String url;
|
||||
private final String eTag;
|
||||
private final String hash;
|
||||
@SerializedName("local")
|
||||
private final long localLastModified;
|
||||
@SerializedName("remote")
|
||||
private final String remoteLastModified;
|
||||
|
||||
/**
|
||||
* For Gson.
|
||||
*/
|
||||
public ETagItem() {
|
||||
this(null, null, null, 0, null);
|
||||
}
|
||||
|
||||
public ETagItem(String url, String eTag, String hash, long localLastModified, String remoteLastModified) {
|
||||
this.url = url;
|
||||
this.eTag = eTag;
|
||||
this.hash = hash;
|
||||
this.localLastModified = localLastModified;
|
||||
this.remoteLastModified = remoteLastModified;
|
||||
}
|
||||
|
||||
public int compareTo(ETagItem other) {
|
||||
if (!url.equals(other.url))
|
||||
throw new IllegalArgumentException();
|
||||
|
||||
ZonedDateTime thisTime = Lang.ignoringException(() -> ZonedDateTime.parse(remoteLastModified, DateTimeFormatter.RFC_1123_DATE_TIME), null);
|
||||
ZonedDateTime otherTime = Lang.ignoringException(() -> ZonedDateTime.parse(other.remoteLastModified, DateTimeFormatter.RFC_1123_DATE_TIME), null);
|
||||
if (thisTime == null && otherTime == null) return 0;
|
||||
else if (thisTime == null) return -1;
|
||||
else if (otherTime == null) return 1;
|
||||
else return thisTime.compareTo(otherTime);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
ETagItem eTagItem = (ETagItem) o;
|
||||
return localLastModified == eTagItem.localLastModified &&
|
||||
Objects.equals(url, eTagItem.url) &&
|
||||
Objects.equals(eTag, eTagItem.eTag) &&
|
||||
Objects.equals(hash, eTagItem.hash) &&
|
||||
Objects.equals(remoteLastModified, eTagItem.remoteLastModified);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(url, eTag, hash, localLastModified, remoteLastModified);
|
||||
}
|
||||
}
|
||||
|
||||
private static CacheRepository instance = new CacheRepository();
|
||||
|
||||
public static CacheRepository getInstance() {
|
||||
|
@ -17,10 +17,7 @@
|
||||
*/
|
||||
package org.jackhuang.hmcl.util.io;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.io.*;
|
||||
import java.nio.charset.Charset;
|
||||
|
||||
/**
|
||||
@ -60,6 +57,14 @@ public final class IOUtils {
|
||||
return readFully(stream).toString(charset.name());
|
||||
}
|
||||
|
||||
public static void write(String text, OutputStream outputStream) throws IOException {
|
||||
write(text.getBytes(), outputStream);
|
||||
}
|
||||
|
||||
public static void write(byte[] bytes, OutputStream outputStream) throws IOException {
|
||||
copyTo(new ByteArrayInputStream(bytes), outputStream);
|
||||
}
|
||||
|
||||
public static void copyTo(InputStream src, OutputStream dest) throws IOException {
|
||||
copyTo(src, dest, new byte[DEFAULT_BUFFER_SIZE]);
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user