Voomra/Project-Mirror
0
Fork 0
Code Issues 2 Releases Activity

WIP

Browse Source
This commit is contained in:
DmitriyMX
2022-05-13 15:25:12 +03:00
parent 6ad5e0c0e7
commit 712e286558
13 changed files with 317 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
mirror-core/build.gradle
View File

@@ -3,4 +3,7 @@ apply plugin: 'java-library'
dependencies {
api("com.squareup.okhttp3:okhttp:${okhttpVersion}")
implementation("commons-io:commons-io:${commonsIoVersion}")
implementation("commons-codec:commons-codec:${commonsCodecVersion}")
testImplementation("org.slf4j:slf4j-simple:${slf4jVersion}")
}

7
mirror-core/src/main/java/ru/di9/mirror/core/dao/ArtifactDao.java
View File

@@ -1,8 +1,15 @@
package ru.di9.mirror.core.dao;
import ru.di9.mirror.core.domain.ArtifactDefinition;
import ru.di9.mirror.core.entity.ArtifactEntity;
import java.util.Optional;
public interface ArtifactDao {
default Optional<ArtifactEntity> findByDefinition(ArtifactDefinition definition) {
return Optional.empty();
}
ArtifactEntity save(ArtifactEntity artifactEntity);
}

31
mirror-core/src/main/java/ru/di9/mirror/core/domain/ArtifactDefinition.java Normal file
View File

@@ -0,0 +1,31 @@
package ru.di9.mirror.core.domain;
import java.util.StringJoiner;
public record ArtifactDefinition(String group, String name, String version) {
private static final String SNAPSHOT = "SNAPSHOT";
public static ArtifactDefinition fromPath(String path) {
if (path.startsWith("/")) {
path = path.substring(1);
}
String[] partsPath = path.split("/");
String version = partsPath[partsPath.length - 2];
String fileName = partsPath[partsPath.length - 1];
String artifactId;
if (version.contains(SNAPSHOT)) {
artifactId = fileName.substring(0, fileName.indexOf(version.substring(0, version.lastIndexOf(SNAPSHOT))) - 1);
} else {
artifactId = fileName.substring(0, (fileName.lastIndexOf('.') - version.length() - 1));
}
StringJoiner group = new StringJoiner(".");
for (int i = 0; i < partsPath.length - 3; i++) {
group.add(partsPath[i]);
}
return new ArtifactDefinition(group.toString(), artifactId, version);
}
}

41
mirror-core/src/main/java/ru/di9/mirror/core/entity/ArtifactEntity.java
View File

@@ -1,49 +1,16 @@
package ru.di9.mirror.core.entity;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import java.util.StringJoiner;
@Slf4j
@AllArgsConstructor
@NoArgsConstructor
@Data
public class ArtifactEntity {
private static final String SNAPSHOT = "SNAPSHOT";
private String group;
private String artifactId;
private String version;
private boolean isSnapshot;
public static ArtifactEntity fromHttpPath(String httpPath) {
if (httpPath.startsWith("/")) {
httpPath = httpPath.substring(1);
}
log.info("httpPath = {}", httpPath);
String[] partsPath = httpPath.split("/");
String version = partsPath[partsPath.length-2];
boolean isSnapshot = version.contains(SNAPSHOT);
String fileName = partsPath[partsPath.length-1];
String artifactId;
if (isSnapshot) {
String partVersion = version.substring(0, version.lastIndexOf(SNAPSHOT));
artifactId = fileName.substring(0, fileName.indexOf(partVersion) - 1);
} else {
artifactId = fileName.substring(0, (fileName.lastIndexOf('.') - version.length() - 1));
}
StringJoiner sj = new StringJoiner(".");
for (int i = 0; i < partsPath.length - 3; i++) {
sj.add(partsPath[i]);
}
return new ArtifactEntity(sj.toString(), artifactId, version, isSnapshot);
}
private String sha1Jar;
private String sha1Pom;
private String store;
}

3
mirror-core/src/main/java/ru/di9/mirror/core/handler/IndexOfHandler.java
View File

@@ -4,6 +4,7 @@ import lombok.RequiredArgsConstructor;
import ru.di9.mirror.core.domain.FileItem;
import ru.di9.mirror.core.storage.FileStorage;
import java.util.Collections;
import java.util.List;
@RequiredArgsConstructor
@@ -12,7 +13,7 @@ public class IndexOfHandler {
private final FileStorage fileStorage;
public List<FileItem> walker(String path) {
return fileStorage.list("/local/" + path);
return Collections.emptyList();//fileStorage.list("/local/" + path);
}
}

261
mirror-core/src/main/java/ru/di9/mirror/core/handler/MavenHandler.java
View File

@@ -2,16 +2,26 @@ package ru.di9.mirror.core.handler;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.codec.digest.MessageDigestAlgorithms;
import org.apache.commons.io.IOUtils;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import ru.di9.mirror.core.domain.ArtifactDefinition;
import ru.di9.mirror.core.domain.FileInfo;
import ru.di9.mirror.core.entity.ArtifactEntity;
import ru.di9.mirror.core.handler.response.GetFileResponse;
import ru.di9.mirror.core.dao.ArtifactDao;
import ru.di9.mirror.core.storage.FileStorage;
import ru.di9.mirror.core.service.ExternalMavenService;
import ru.di9.mirror.core.util.FileInfoWrapper;
import ru.di9.mirror.core.util.Utils;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.util.List;
import java.util.Optional;
@@ -19,75 +29,212 @@ import java.util.Optional;
@RequiredArgsConstructor
public class MavenHandler {
private final ArtifactDao artifactDao;
private final FileStorage fileStorage;
private final List<ExternalMavenService> externalMavenServices;
private final ArtifactDao repository;
public Optional<GetFileResponse> getFile(String path) {
var fileInfo = FileInfo.of(path.substring(path.lastIndexOf("/") + 1));
ArtifactEntity artifactEntity = null;
if (!fileInfo.fullName().equalsIgnoreCase("maven-metadata.xml")
&& !fileInfo.ext().equalsIgnoreCase("sha1")) {
artifactEntity = ArtifactEntity.fromHttpPath(path);
log.info(artifactEntity.toString());
public Optional<GetFileResponse> getArtifactFile(String path) {
var fileInfo = new FileInfoWrapper(path.substring(path.lastIndexOf("/") + 1));
String ext = fileInfo.ext();
if (!ext.equalsIgnoreCase("jar")
&& !ext.equalsIgnoreCase("pom")
&& !ext.equalsIgnoreCase("sha1")) {
return Optional.empty();
}
Optional<InputStream> optionalInputStream = fileStorage.findByName("/local/" + path);
if (optionalInputStream.isPresent()) {
return optionalInputStream
.map(inputStream -> new GetFileResponse(fileInfo.name(), inputStream));
var artifactDefinition = ArtifactDefinition.fromPath(path);
Optional<ArtifactEntity> entity = artifactDao.findByDefinition(artifactDefinition);
if (entity.isPresent()) {
return processExistsArtifact(ext, path, fileInfo, entity.get());
} else {
return findInMirrors(path, fileInfo, artifactEntity);
}
}
private Optional<GetFileResponse> findInMirrors(String path, FileInfo fileInfo, ArtifactEntity artifactEntity) {
Optional<InputStream> result;
for (ExternalMavenService externalMavenService : externalMavenServices) {
final String nameForStore = "/" + externalMavenService.getId() + "/" + path;
result = fileStorage.findByName(nameForStore);
if (result.isPresent()) {
return result
.map(inputStream -> new GetFileResponse(fileInfo.name(), inputStream));
} else {
result = externalMavenService.getFile(path);
if (result.isPresent()) {
if (artifactEntity != null) {
repository.save(artifactEntity);
}
if (fileInfo.ext().equalsIgnoreCase("sha1")) {
try {
String sha1str = IOUtils.toString(result.get(), StandardCharsets.UTF_8);
fileStorage.setMetadata(nameForStore.substring(0, nameForStore.lastIndexOf(".")), "sha1", sha1str);
} catch (Exception e) {
throw new RuntimeException(e);
}
} else {
fileStorage.save(nameForStore, result.get());
}
return fileStorage.findByName(nameForStore)
.map(inputStream -> new GetFileResponse(fileInfo.name(), inputStream));
}
if (ext.equalsIgnoreCase("jar") || ext.equalsIgnoreCase("pom")) {
return processJarPom2(path, fileInfo, ext, artifactDefinition);
} else if (ext.equalsIgnoreCase("sha1")) {
return Optional.empty();
}
}
return Optional.empty();
}
private Optional<GetFileResponse> processJarPom2(String path, FileInfoWrapper fileInfo, String ext, ArtifactDefinition artifactDefinition) {
for (ExternalMavenService externalMavenService : externalMavenServices) {
Optional<InputStream> inputStream = externalMavenService.getFile(path);
if (inputStream.isEmpty()) {
continue;
}
byte[] bytes;
try {
bytes = IOUtils.toByteArray(inputStream.get());
inputStream.get().close();
} catch (IOException e) {
log.error(e.getMessage());
throw new RuntimeException(e);
}
fileStorage.put(path, new ByteArrayInputStream(bytes));
var artifactEntity = new ArtifactEntity();
artifactEntity.setGroup(artifactDefinition.group());
artifactEntity.setArtifactId(artifactDefinition.name());
artifactEntity.setVersion(artifactDefinition.version());
artifactEntity.setStore(externalMavenService.getId());
setSha1(ext, artifactEntity, new DigestUtils(MessageDigestAlgorithms.SHA_1).digestAsHex(bytes));
artifactDao.save(artifactEntity);
return Optional.of(new GetFileResponse(fileInfo.fullName(), new ByteArrayInputStream(bytes)));
}
return Optional.empty();
}
private Optional<GetFileResponse> processExistsArtifact(String ext, String path,
FileInfoWrapper fileInfo, ArtifactEntity entity) {
if (ext.equalsIgnoreCase("jar") || ext.equalsIgnoreCase("pom")) {
return processJarPom(path, fileInfo.fullName(), ext, entity);
} else if (ext.equalsIgnoreCase("sha1")) {
return processSha1(fileInfo, entity);
}
return Optional.empty();
}
private Optional<GetFileResponse> processSha1(FileInfoWrapper fileInfo, ArtifactEntity entity) {
var fileInfo2 = new FileInfoWrapper(fileInfo.name());
if (fileInfo2.ext().equalsIgnoreCase("jar")) {
return Optional.of(new GetFileResponse(fileInfo.fullName(), IOUtils.toInputStream(entity.getSha1Jar(), StandardCharsets.UTF_8)));
} else if (fileInfo2.ext().equalsIgnoreCase("pom")) {
return Optional.of(new GetFileResponse(fileInfo.fullName(), IOUtils.toInputStream(entity.getSha1Pom(), StandardCharsets.UTF_8)));
} else {
return Optional.empty();
}
}
private Optional<GetFileResponse> processJarPom(String path, String fullFileName, String ext, ArtifactEntity entity) {
if (Utils.isNotEmptyString(getSha1(ext, entity))) {
return Optional.of(new GetFileResponse(fullFileName, fileStorage.get(path)));
}
Optional<ExternalMavenService> externalMavenService = externalMavenServices.stream()
.filter(ems -> ems.getId().equalsIgnoreCase(entity.getStore()))
.findFirst();
if (externalMavenService.isPresent()) {
return getFileFromExternalMavenService(path, fullFileName, ext, entity, externalMavenService.get());
} else {
log.warn("Unknown store: {}", entity.getStore());
return Optional.empty();
}
}
private Optional<GetFileResponse> getFileFromExternalMavenService(String path, String fullFileName, String ext, ArtifactEntity entity, ExternalMavenService externalMavenService) {
return externalMavenService.getFile(path)
.map(inputStream -> {
byte[] bytes = null;
try {
bytes = IOUtils.toByteArray(inputStream);
inputStream.close();
} catch (IOException e) {
log.error(e.getMessage(), e);
}
return bytes;
})
.map(bytes -> {
fileStorage.put(path, new ByteArrayInputStream(bytes));
setSha1(ext, entity, new DigestUtils(MessageDigestAlgorithms.SHA_1).digestAsHex(bytes));
artifactDao.save(entity);
return new GetFileResponse(fullFileName, new ByteArrayInputStream(bytes));
});
}
private String getSha1(String type, ArtifactEntity entity) {
if (type.equalsIgnoreCase("jar")) {
return entity.getSha1Jar();
} else if (type.equalsIgnoreCase("pom")) {
return entity.getSha1Pom();
} else {
return null;
}
}
private void setSha1(String type, ArtifactEntity entity, String value) {
if (type.equalsIgnoreCase("jar")) {
entity.setSha1Jar(value);
} else if (type.equalsIgnoreCase("pom")) {
entity.setSha1Pom(value);
}
}
public Optional<GetFileResponse> getFile(String path) {
// var fileInfo = FileInfo.of(path.substring(path.lastIndexOf("/") + 1));
//
// ArtifactEntity artifactEntity = null;
// if (!fileInfo.fullName().equalsIgnoreCase("maven-metadata.xml")
// && !fileInfo.ext().equalsIgnoreCase("sha1")) {
//
// artifactEntity = ArtifactEntity.fromHttpPath(path);
// log.info(artifactEntity.toString());
// }
//
// Optional<InputStream> optionalInputStream = fileStorage.findByName("/local/" + path);
// if (optionalInputStream.isPresent()) {
// return optionalInputStream
// .map(inputStream -> new GetFileResponse(fileInfo.name(), inputStream));
// } else {
// return findInMirrors(path, fileInfo, artifactEntity);
// }
return null;
}
private Optional<GetFileResponse> findInMirrors(String path, FileInfo fileInfo, ArtifactEntity artifactEntity) {
// Optional<InputStream> result;
// for (ExternalMavenService externalMavenService : externalMavenServices) {
// final String nameForStore = "/" + externalMavenService.getId() + "/" + path;
//
// result = fileStorage.findByName(nameForStore);
// if (result.isPresent()) {
// return result
// .map(inputStream -> new GetFileResponse(fileInfo.name(), inputStream));
// } else {
// result = externalMavenService.getFile(path);
// if (result.isPresent()) {
// if (artifactEntity != null) {
// artifactDao.save(artifactEntity);
// }
//
// if (fileInfo.ext().equalsIgnoreCase("sha1")) {
// try {
// String sha1str = IOUtils.toString(result.get(), StandardCharsets.UTF_8);
// fileStorage.setMetadata(nameForStore.substring(0, nameForStore.lastIndexOf(".")), "sha1", sha1str);
// } catch (Exception e) {
// throw new RuntimeException(e);
// }
// } else {
// fileStorage.save(nameForStore, result.get());
// }
// return fileStorage.findByName(nameForStore)
// .map(inputStream -> new GetFileResponse(fileInfo.name(), inputStream));
// }
// }
// }
return Optional.empty();
}
public void putFile(String path, InputStream inputStream) {
var fileInfo = FileInfo.of(path.substring(path.lastIndexOf("/") + 1));
if (!fileInfo.fullName().equalsIgnoreCase("maven-metadata.xml")
&& !fileInfo.ext().equalsIgnoreCase("sha1")) {
ArtifactEntity artifactEntity = ArtifactEntity.fromHttpPath(path);
log.info(artifactEntity.toString());
}
fileStorage.save("/local/" + path, inputStream);
// var fileInfo = FileInfo.of(path.substring(path.lastIndexOf("/") + 1));
//
// if (!fileInfo.fullName().equalsIgnoreCase("maven-metadata.xml")
// && !fileInfo.ext().equalsIgnoreCase("sha1")) {
//
// ArtifactEntity artifactEntity = ArtifactEntity.fromHttpPath(path);
// log.info(artifactEntity.toString());
// }
//
// fileStorage.save("/local/" + path, inputStream);
}
}

12
mirror-core/src/main/java/ru/di9/mirror/core/storage/FileStorage.java
View File

@@ -1,18 +1,10 @@
package ru.di9.mirror.core.storage;
import ru.di9.mirror.core.domain.FileItem;
import java.io.InputStream;
import java.util.List;
import java.util.Optional;
public interface FileStorage {
Optional<InputStream> findByName(String name);
InputStream get(String name);
void save(String name, InputStream inputStream);
void setMetadata(String name, String key, String value);
List<FileItem> list(String prefix);
void put(String name, InputStream inputStream);
}

12
mirror-core/src/main/java/ru/di9/mirror/core/util/FileInfoWrapper.java Normal file
View File

@@ -0,0 +1,12 @@
package ru.di9.mirror.core.util;
public record FileInfoWrapper(String fullName) {
public String name() {
return fullName.substring(0, fullName.lastIndexOf('.'));
}
public String ext() {
return fullName.substring(fullName.lastIndexOf('.') + 1);
}
}

2
mirror-core/src/main/java/ru/di9/mirror/core/Utils.java → mirror-core/src/main/java/ru/di9/mirror/core/util/Utils.java
View File

@@ -1,4 +1,4 @@
package ru.di9.mirror.core;
package ru.di9.mirror.core.util;
import lombok.experimental.UtilityClass;

58
mirror-core/src/test/java/ru/di9/mirror/core/entity/ArtifactEntityTest.java
View File

@@ -1,29 +1,29 @@
package ru.di9.mirror.core.entity;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import java.util.stream.Stream;
class ArtifactEntityTest {
@ParameterizedTest(name = "[{index}] {0}")
@MethodSource("fromHttpPathSource")
void fromHttpPath(String httpPath, ArtifactEntity expected) {
ArtifactEntity actual = ArtifactEntity.fromHttpPath(httpPath);
Assertions.assertEquals(expected, actual);
}
static Stream<Arguments> fromHttpPathSource() {
return Stream.of(
Arguments.of("com/google/guava/guava/21.0/guava-21.0.jar",
new ArtifactEntity("com.google.guava", "guava", "21.0", false)),
Arguments.of("org/bukkit/bukkit/1.12.2-R0.1-SNAPSHOT/bukkit-1.12.2-R0.1-20180712.012114-155.jar",
new ArtifactEntity("org.bukkit", "bukkit", "1.12.2-R0.1-SNAPSHOT", true)),
Arguments.of("/org/projectlombok/lombok/1.18.22/lombok-1.18.22.jar",
new ArtifactEntity("org.projectlombok", "lombok", "1.18.22", false))
);
}
}
//package ru.di9.mirror.core.entity;
//
//import org.junit.jupiter.api.Assertions;
//import org.junit.jupiter.params.ParameterizedTest;
//import org.junit.jupiter.params.provider.Arguments;
//import org.junit.jupiter.params.provider.MethodSource;
//
//import java.util.stream.Stream;
//
//class ArtifactEntityTest {
//
// @ParameterizedTest(name = "[{index}] {0}")
// @MethodSource("fromHttpPathSource")
// void fromHttpPath(String httpPath, ArtifactEntity expected) {
// ArtifactEntity actual = ArtifactEntity.fromHttpPath(httpPath);
// Assertions.assertEquals(expected, actual);
// }
//
// static Stream<Arguments> fromHttpPathSource() {
// return Stream.of(
// Arguments.of("com/google/guava/guava/21.0/guava-21.0.jar",
// new ArtifactEntity("com.google.guava", "guava", "21.0", false)),
// Arguments.of("org/bukkit/bukkit/1.12.2-R0.1-SNAPSHOT/bukkit-1.12.2-R0.1-20180712.012114-155.jar",
// new ArtifactEntity("org.bukkit", "bukkit", "1.12.2-R0.1-SNAPSHOT", true)),
// Arguments.of("/org/projectlombok/lombok/1.18.22/lombok-1.18.22.jar",
// new ArtifactEntity("org.projectlombok", "lombok", "1.18.22", false))
// );
// }
//}

24
mirror-core/src/test/java/ru/di9/mirror/core/handler/MavenHandlerTest.java Normal file
View File

@@ -0,0 +1,24 @@
package ru.di9.mirror.core.handler;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import ru.di9.mirror.core.handler.response.GetFileResponse;
import java.util.Optional;
import static org.junit.jupiter.api.Assertions.*;
class MavenHandlerTest {
@Test
@DisplayName("В БД есть артефакт")
void existsArtifact() {
MavenHandler mavenHandler = new MavenHandler(null, null, null);
Optional<GetFileResponse> artifactFile = mavenHandler.getArtifactFile("ghast/ghast-tools/1.13/ghast-tools-1.13.jar");
assertNotNull(artifactFile);
assertTrue(artifactFile.isPresent());
assertEquals("ghast-tools-1.13.jar", artifactFile.get().name());
assertNotNull(artifactFile.get().inputStream());
}
}

2
mirror-web/src/main/java/ru/di9/mirror/web/config/WebConfig.java
View File

@@ -18,7 +18,7 @@ public class WebConfig {
public MavenHandler mavenHandler(FileStorage fileStorage,
List<ExternalMavenService> externalMavenServices,
ArtifactDao artifactDao) {
return new MavenHandler(fileStorage, externalMavenServices, artifactDao);
return new MavenHandler(artifactDao, fileStorage, externalMavenServices);
}
@Bean

1
versions.gradle
View File

@@ -6,6 +6,7 @@ ext {
mongoDriver = '4.6.0'
okhttpVersion = '4.8.1'
commonsIoVersion = '2.11.0'
commonsCodecVersion = '1.15'
// for tests only
junitJupiterVersion = '5.5.2'