From cbfe2c50a721830563de12607a34504076fcd7ae Mon Sep 17 00:00:00 2001 From: DmitriyMX Date: Fri, 29 Apr 2022 12:35:14 +0300 Subject: [PATCH] refactory --- build.gradle | 16 ++- mirror-app/build.gradle | 5 - .../ru/di9/mirror/app/domain/FileRecord.java | 6 - .../ru/di9/mirror/app/domain/PathRecord.java | 4 - .../di9/mirror/app/domain/WalkerResult.java | 7 -- .../app/service/FileStorageService.java | 112 ----------------- .../service/MavenCentralStorageService.java | 44 ------- .../di9/mirror/app/service/MirrorService.java | 19 --- .../app/service/MirroredStorageService.java | 52 -------- .../mirror/app/service/StorageService.java | 16 --- mirror-core/build.gradle | 33 +++++ .../core/service/MinioServiceIntTest.java | 68 +++++++++++ .../main/java/ru/di9/mirror/core}/Utils.java | 6 +- .../di9/mirror/core/domain/ItemRecord2.java | 28 +++++ .../domain/MavenHandlerGetFileResponse.java | 6 + .../ru/di9/mirror/core}/domain/PathType.java | 2 +- .../di9/mirror/core/domain/WalkerResult.java | 5 + .../mirror/core/handler/IndexOfHandler.java | 18 +++ .../di9/mirror/core/handler/MavenHandler.java | 50 ++++++++ .../core/service/ExternalMavenService.java | 55 +++++++++ .../di9/mirror/core/service/MinioService.java | 68 +++++++++++ .../mirror/core/service/MinioServiceTest.java | 63 ++++++++++ mirror-web/build.gradle | 9 +- .../web/config/MavenMirrorsProperties.java | 20 ++++ .../ru/di9/mirror/web/config/WebConfig.java | 46 +++++-- .../controller/IndexOfMavenController.java | 113 ++++++++---------- .../web/controller/MavenController.java | 35 ++---- ...itional-spring-configuration-metadata.json | 32 ++++- mirror-web/src/main/resources/application.yml | 19 ++- .../src/main/resources/templates/list.ftl | 19 +++ settings.gradle | 2 +- versions.gradle | 8 +- 32 files changed, 603 insertions(+), 383 deletions(-) delete mode 100644 mirror-app/build.gradle delete mode 100644 mirror-app/src/main/java/ru/di9/mirror/app/domain/FileRecord.java delete mode 100644 mirror-app/src/main/java/ru/di9/mirror/app/domain/PathRecord.java delete mode 100644 mirror-app/src/main/java/ru/di9/mirror/app/domain/WalkerResult.java delete mode 100644 mirror-app/src/main/java/ru/di9/mirror/app/service/FileStorageService.java delete mode 100644 mirror-app/src/main/java/ru/di9/mirror/app/service/MavenCentralStorageService.java delete mode 100644 mirror-app/src/main/java/ru/di9/mirror/app/service/MirrorService.java delete mode 100644 mirror-app/src/main/java/ru/di9/mirror/app/service/MirroredStorageService.java delete mode 100644 mirror-app/src/main/java/ru/di9/mirror/app/service/StorageService.java create mode 100644 mirror-core/build.gradle create mode 100644 mirror-core/src/integrationTest/java/ru/di9/mirror/core/service/MinioServiceIntTest.java rename {mirror-app/src/main/java/ru/di9/mirror/app => mirror-core/src/main/java/ru/di9/mirror/core}/Utils.java (67%) create mode 100644 mirror-core/src/main/java/ru/di9/mirror/core/domain/ItemRecord2.java create mode 100644 mirror-core/src/main/java/ru/di9/mirror/core/domain/MavenHandlerGetFileResponse.java rename {mirror-app/src/main/java/ru/di9/mirror/app => mirror-core/src/main/java/ru/di9/mirror/core}/domain/PathType.java (56%) create mode 100644 mirror-core/src/main/java/ru/di9/mirror/core/domain/WalkerResult.java create mode 100644 mirror-core/src/main/java/ru/di9/mirror/core/handler/IndexOfHandler.java create mode 100644 mirror-core/src/main/java/ru/di9/mirror/core/handler/MavenHandler.java create mode 100644 mirror-core/src/main/java/ru/di9/mirror/core/service/ExternalMavenService.java create mode 100644 mirror-core/src/main/java/ru/di9/mirror/core/service/MinioService.java create mode 100644 mirror-core/src/test/java/ru/di9/mirror/core/service/MinioServiceTest.java create mode 100644 mirror-web/src/main/java/ru/di9/mirror/web/config/MavenMirrorsProperties.java create mode 100644 mirror-web/src/main/resources/templates/list.ftl diff --git a/build.gradle b/build.gradle index 7e20999..1c11d9d 100644 --- a/build.gradle +++ b/build.gradle @@ -7,9 +7,7 @@ buildscript { } dependencies { // https://plugins.gradle.org/plugin/org.springframework.boot - classpath "org.springframework.boot:spring-boot-gradle-plugin:${pluginSpringBootVerson}" - // https://plugins.gradle.org/plugin/io.spring.dependency-management - classpath "io.spring.gradle:dependency-management-plugin:${pluginSpringBomVerson}" + classpath "org.springframework.boot:spring-boot-gradle-plugin:${springBootVerson}" } } @@ -29,9 +27,15 @@ subprojects { dependencies { annotationProcessor("org.projectlombok:lombok:${lombokVersion}") compileOnly("org.projectlombok:lombok:${lombokVersion}") + + testAnnotationProcessor("org.projectlombok:lombok:${lombokVersion}") + testCompileOnly("org.projectlombok:lombok:${lombokVersion}") + testImplementation("org.junit.jupiter:junit-jupiter-api:${junitJupiterVersion}") + testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:${junitJupiterVersion}") + testImplementation("org.mockito:mockito-core:${mockitoCoreVersion}") } - configurations.implementation.resolutionStrategy { - failOnVersionConflict() - } + test { + useJUnitPlatform() + } } diff --git a/mirror-app/build.gradle b/mirror-app/build.gradle deleted file mode 100644 index c5d6b4c..0000000 --- a/mirror-app/build.gradle +++ /dev/null @@ -1,5 +0,0 @@ -apply plugin: 'java-library' - -dependencies { - api("com.squareup.okhttp3:okhttp:${okhttpVersion}") -} diff --git a/mirror-app/src/main/java/ru/di9/mirror/app/domain/FileRecord.java b/mirror-app/src/main/java/ru/di9/mirror/app/domain/FileRecord.java deleted file mode 100644 index 0fd0bf8..0000000 --- a/mirror-app/src/main/java/ru/di9/mirror/app/domain/FileRecord.java +++ /dev/null @@ -1,6 +0,0 @@ -package ru.di9.mirror.app.domain; - -import java.io.InputStream; - -public record FileRecord(String name, long size, InputStream inputStream) { -} diff --git a/mirror-app/src/main/java/ru/di9/mirror/app/domain/PathRecord.java b/mirror-app/src/main/java/ru/di9/mirror/app/domain/PathRecord.java deleted file mode 100644 index 9f9e914..0000000 --- a/mirror-app/src/main/java/ru/di9/mirror/app/domain/PathRecord.java +++ /dev/null @@ -1,4 +0,0 @@ -package ru.di9.mirror.app.domain; - -public record PathRecord(String name, PathType type) { -} diff --git a/mirror-app/src/main/java/ru/di9/mirror/app/domain/WalkerResult.java b/mirror-app/src/main/java/ru/di9/mirror/app/domain/WalkerResult.java deleted file mode 100644 index 646490c..0000000 --- a/mirror-app/src/main/java/ru/di9/mirror/app/domain/WalkerResult.java +++ /dev/null @@ -1,7 +0,0 @@ -package ru.di9.mirror.app.domain; - -import java.util.List; - -public record WalkerResult(PathType pathType, FileRecord fileRecord, List filesList) { - -} diff --git a/mirror-app/src/main/java/ru/di9/mirror/app/service/FileStorageService.java b/mirror-app/src/main/java/ru/di9/mirror/app/service/FileStorageService.java deleted file mode 100644 index d6a4e01..0000000 --- a/mirror-app/src/main/java/ru/di9/mirror/app/service/FileStorageService.java +++ /dev/null @@ -1,112 +0,0 @@ -package ru.di9.mirror.app.service; - -import ru.di9.mirror.app.Utils; -import ru.di9.mirror.app.domain.FileRecord; -import ru.di9.mirror.app.domain.PathRecord; -import ru.di9.mirror.app.domain.PathType; -import ru.di9.mirror.app.domain.WalkerResult; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.stream.Stream; - -public class FileStorageService implements StorageService { - - private final Path storagePath; - - public FileStorageService(String storagePath) { - this.storagePath = Paths.get(storagePath).resolve("local"); - } - - @SuppressWarnings("java:S112") - @Override - public Optional getFile(String path) { - Path fullPath = storageResolve(path); - - if (Files.notExists(fullPath)) { - return Optional.empty(); - } - - try { - return Optional.of(new FileRecord( - fullPath.toFile().getName(), - Files.size(fullPath), - Files.newInputStream(fullPath) - )); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @SuppressWarnings("java:S112") - @Override - public void putFile(String path, InputStream inputStream) { - Path fullPath = storageResolve(path); - try { - Files.createDirectories(fullPath.getParent()); - } catch (IOException e) { - throw new RuntimeException(e); - } - - try (OutputStream outputStream = Files.newOutputStream(fullPath)) { - byte[] buff = new byte[8 * 1024]; - int len; - while ((len = inputStream.read(buff)) > 0) { - outputStream.write(buff, 0, len); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @SuppressWarnings("java:S112") - @Override - public Optional walker(String path) { - Path fullPath = storageResolve(path); - - if (Files.notExists(fullPath)) { - return Optional.empty(); - } - - if (Files.isDirectory(fullPath)) { - final List list = new ArrayList<>(); - - try (Stream walk = Files.walk(fullPath, 1)) { - walk.skip(1).forEach(path1 -> list.add(new PathRecord( - path1.getFileName().toString(), - Files.isDirectory(path1) ? PathType.DIRECTORY : PathType.FILE))); - } catch (IOException e) { - throw new RuntimeException(e); - } - - return Optional.of(new WalkerResult(PathType.DIRECTORY, null, list)); - } else { - try { - var fileRecord = new FileRecord( - fullPath.getFileName().toString(), - Files.size(fullPath), - Files.newInputStream(fullPath) - ); - - return Optional.of(new WalkerResult(PathType.FILE, fileRecord, null)); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - } - - private Path storageResolve(String httpPath) { - if (Utils.isEmptyString(httpPath)) { - return storagePath; - } else { - return storagePath.resolve(httpPath.substring(1)); - } - } -} diff --git a/mirror-app/src/main/java/ru/di9/mirror/app/service/MavenCentralStorageService.java b/mirror-app/src/main/java/ru/di9/mirror/app/service/MavenCentralStorageService.java deleted file mode 100644 index b24e4eb..0000000 --- a/mirror-app/src/main/java/ru/di9/mirror/app/service/MavenCentralStorageService.java +++ /dev/null @@ -1,44 +0,0 @@ -package ru.di9.mirror.app.service; - -import lombok.SneakyThrows; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.Response; -import ru.di9.mirror.app.domain.FileRecord; - -import java.io.IOException; -import java.util.Objects; -import java.util.Optional; - -public class MavenCentralStorageService extends MirrorService { - - private static final String MAVEN_CENTRAL_URL = "https://repo1.maven.org/maven2"; - private final OkHttpClient okHttpClient; - - public MavenCentralStorageService(OkHttpClient okHttpClient) { - this.okHttpClient = okHttpClient; - } - - @Override - @SneakyThrows - public Optional getFile(String path) { - Request request = new Request.Builder() - .url(MAVEN_CENTRAL_URL + path) - .build(); - - Response response = okHttpClient.newCall(request).execute(); - if (!response.isSuccessful()) { - if (response.code() == 404) { - return Optional.empty(); - } - - throw new IOException("Unexpected code " + response); - } - - return Optional.of(new FileRecord( - path, - Long.parseLong(Objects.requireNonNull(response.header("Content-Length"))), - Objects.requireNonNull(response.body()).byteStream() - )); - } -} diff --git a/mirror-app/src/main/java/ru/di9/mirror/app/service/MirrorService.java b/mirror-app/src/main/java/ru/di9/mirror/app/service/MirrorService.java deleted file mode 100644 index 97f9794..0000000 --- a/mirror-app/src/main/java/ru/di9/mirror/app/service/MirrorService.java +++ /dev/null @@ -1,19 +0,0 @@ -package ru.di9.mirror.app.service; - -import ru.di9.mirror.app.domain.WalkerResult; - -import java.io.InputStream; -import java.util.Optional; - -public abstract class MirrorService implements StorageService{ - - @Override - public void putFile(String path, InputStream inputStream) { - throw new UnsupportedOperationException(); - } - - @Override - public Optional walker(String path) { - throw new UnsupportedOperationException(); - } -} diff --git a/mirror-app/src/main/java/ru/di9/mirror/app/service/MirroredStorageService.java b/mirror-app/src/main/java/ru/di9/mirror/app/service/MirroredStorageService.java deleted file mode 100644 index 6fb3b3f..0000000 --- a/mirror-app/src/main/java/ru/di9/mirror/app/service/MirroredStorageService.java +++ /dev/null @@ -1,52 +0,0 @@ -package ru.di9.mirror.app.service; - -import ru.di9.mirror.app.domain.FileRecord; -import ru.di9.mirror.app.domain.WalkerResult; - -import java.io.InputStream; -import java.util.List; -import java.util.Optional; - -@SuppressWarnings("ClassCanBeRecord") -public class MirroredStorageService implements StorageService { - - private final FileStorageService fileStorageService; - private final List mirrors; - - public MirroredStorageService(FileStorageService fileStorageService, - List mirrors) { - this.fileStorageService = fileStorageService; - this.mirrors = mirrors; - } - - @Override - public Optional getFile(String path) { - Optional optional; - - optional = fileStorageService.getFile(path); - if (optional.isPresent()) { - return optional; - } - - for (StorageService mirror : mirrors) { - optional = mirror.getFile(path); - if (optional.isPresent()) { - FileRecord fileRecord = optional.get(); - fileStorageService.putFile(fileRecord.name(), fileRecord.inputStream()); - break; - } - } - - return fileStorageService.getFile(path); - } - - @Override - public void putFile(String path, InputStream inputStream) { - fileStorageService.putFile(path, inputStream); - } - - @Override - public Optional walker(String path) { - return fileStorageService.walker(path); - } -} diff --git a/mirror-app/src/main/java/ru/di9/mirror/app/service/StorageService.java b/mirror-app/src/main/java/ru/di9/mirror/app/service/StorageService.java deleted file mode 100644 index 5ed475b..0000000 --- a/mirror-app/src/main/java/ru/di9/mirror/app/service/StorageService.java +++ /dev/null @@ -1,16 +0,0 @@ -package ru.di9.mirror.app.service; - -import ru.di9.mirror.app.domain.FileRecord; -import ru.di9.mirror.app.domain.WalkerResult; - -import java.io.InputStream; -import java.util.Optional; - -public interface StorageService { - - Optional getFile(String path); - - void putFile(String path, InputStream inputStream); - - Optional walker(String path); -} diff --git a/mirror-core/build.gradle b/mirror-core/build.gradle new file mode 100644 index 0000000..4b2369c --- /dev/null +++ b/mirror-core/build.gradle @@ -0,0 +1,33 @@ +apply plugin: 'java-library' +apply plugin: 'idea' + +configurations { + integrationTestImplementation { + extendsFrom(testImplementation) + } + integrationTestRuntimeOnly { + extendsFrom(testRuntimeOnly) + } +} + +sourceSets { + integrationTest { + java { + compileClasspath += main.output + test.output + runtimeClasspath += main.output + test.output + srcDir file('src/integrationTest/java') + } + resources { + srcDir file('src/integrationTest/resources') + } + } +} + +idea.module { + testSourceDirs += sourceSets.integrationTest.java.srcDirs + testResourceDirs += sourceSets.integrationTest.resources.srcDirs +} + +dependencies { + api("io.minio:minio:${minioVersion}") +} diff --git a/mirror-core/src/integrationTest/java/ru/di9/mirror/core/service/MinioServiceIntTest.java b/mirror-core/src/integrationTest/java/ru/di9/mirror/core/service/MinioServiceIntTest.java new file mode 100644 index 0000000..43aedbe --- /dev/null +++ b/mirror-core/src/integrationTest/java/ru/di9/mirror/core/service/MinioServiceIntTest.java @@ -0,0 +1,68 @@ +package ru.di9.mirror.core.service; + +import io.minio.MinioClient; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import ru.di9.mirror.core.domain.ItemRecord2; + +import java.io.InputStream; +import java.util.List; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.*; + +class MinioServiceIntTest { + static final String url = "http://dev.di9.ru:9000"; + static final String accessKey = "mirror"; + static final String secretKey = "mirror123"; + static final String bucket = "mirror"; + static MinioClient minioClient; + MinioService minioService; + + @BeforeAll + static void beforeAll() { + minioClient = MinioClient.builder() + .endpoint(url) + .credentials(accessKey, secretKey) + .build(); + } + + @BeforeEach + void setUp() { + minioService = new MinioService(minioClient, bucket); + } + + @Test + void list() { + List list = minioService.list("/local/ghast/ghast-tools/"); + + assertNotNull(list); + assertFalse(list.isEmpty()); + + assertTrue(list.stream() + .anyMatch(itemRecord2 -> itemRecord2.name().equals("maven-metadata.xml") + && !itemRecord2.isDir() + && itemRecord2.size() == 320)); + assertTrue(list.stream() + .anyMatch(itemRecord2 -> itemRecord2.name().equals("1.13") + && itemRecord2.isDir() + && itemRecord2.size() == 0)); + } + + @Test + void notExists() { + Optional optional = minioService.get("/not/exists/path/to/object"); + + assertNotNull(optional); + assertTrue(optional.isEmpty()); + } + + @Test + void exists() { + Optional optional = minioService.get("/local/ghast/ghast-tools/maven-metadata.xml"); + + assertNotNull(optional); + assertTrue(optional.isPresent()); + } +} diff --git a/mirror-app/src/main/java/ru/di9/mirror/app/Utils.java b/mirror-core/src/main/java/ru/di9/mirror/core/Utils.java similarity index 67% rename from mirror-app/src/main/java/ru/di9/mirror/app/Utils.java rename to mirror-core/src/main/java/ru/di9/mirror/core/Utils.java index 17b2ae4..4cadcdb 100644 --- a/mirror-app/src/main/java/ru/di9/mirror/app/Utils.java +++ b/mirror-core/src/main/java/ru/di9/mirror/core/Utils.java @@ -1,4 +1,4 @@ -package ru.di9.mirror.app; +package ru.di9.mirror.core; import lombok.AccessLevel; import lombok.NoArgsConstructor; @@ -9,4 +9,8 @@ public final class Utils { public static boolean isEmptyString(String string) { return string == null || string.isEmpty() || string.isBlank(); } + + public static boolean isNotEmptyString(String string) { + return !isEmptyString(string); + } } diff --git a/mirror-core/src/main/java/ru/di9/mirror/core/domain/ItemRecord2.java b/mirror-core/src/main/java/ru/di9/mirror/core/domain/ItemRecord2.java new file mode 100644 index 0000000..9617232 --- /dev/null +++ b/mirror-core/src/main/java/ru/di9/mirror/core/domain/ItemRecord2.java @@ -0,0 +1,28 @@ +package ru.di9.mirror.core.domain; + +import io.minio.Result; +import io.minio.messages.Item; +import lombok.SneakyThrows; + +public record ItemRecord2(Result itemResult, String prefix) { + + @SneakyThrows + public boolean isDir() { + return itemResult.get().isDir(); + } + + @SneakyThrows + public String name() { + String objectName = itemResult.get().objectName(); + if (isDir()) { + return objectName.substring(prefix.length() - 1, objectName.length() - 1); + } else { + return objectName.substring(prefix.length() - 1); + } + } + + @SneakyThrows + public long size() { + return itemResult.get().size(); + } +} diff --git a/mirror-core/src/main/java/ru/di9/mirror/core/domain/MavenHandlerGetFileResponse.java b/mirror-core/src/main/java/ru/di9/mirror/core/domain/MavenHandlerGetFileResponse.java new file mode 100644 index 0000000..00e2818 --- /dev/null +++ b/mirror-core/src/main/java/ru/di9/mirror/core/domain/MavenHandlerGetFileResponse.java @@ -0,0 +1,6 @@ +package ru.di9.mirror.core.domain; + +import java.io.InputStream; + +public record MavenHandlerGetFileResponse(String name, InputStream inputStream) { +} diff --git a/mirror-app/src/main/java/ru/di9/mirror/app/domain/PathType.java b/mirror-core/src/main/java/ru/di9/mirror/core/domain/PathType.java similarity index 56% rename from mirror-app/src/main/java/ru/di9/mirror/app/domain/PathType.java rename to mirror-core/src/main/java/ru/di9/mirror/core/domain/PathType.java index 33a66a7..82f7fbb 100644 --- a/mirror-app/src/main/java/ru/di9/mirror/app/domain/PathType.java +++ b/mirror-core/src/main/java/ru/di9/mirror/core/domain/PathType.java @@ -1,4 +1,4 @@ -package ru.di9.mirror.app.domain; +package ru.di9.mirror.core.domain; public enum PathType { FILE, DIRECTORY diff --git a/mirror-core/src/main/java/ru/di9/mirror/core/domain/WalkerResult.java b/mirror-core/src/main/java/ru/di9/mirror/core/domain/WalkerResult.java new file mode 100644 index 0000000..08db874 --- /dev/null +++ b/mirror-core/src/main/java/ru/di9/mirror/core/domain/WalkerResult.java @@ -0,0 +1,5 @@ +package ru.di9.mirror.core.domain; + +public record WalkerResult(PathType type) { + +} diff --git a/mirror-core/src/main/java/ru/di9/mirror/core/handler/IndexOfHandler.java b/mirror-core/src/main/java/ru/di9/mirror/core/handler/IndexOfHandler.java new file mode 100644 index 0000000..5c26518 --- /dev/null +++ b/mirror-core/src/main/java/ru/di9/mirror/core/handler/IndexOfHandler.java @@ -0,0 +1,18 @@ +package ru.di9.mirror.core.handler; + +import lombok.RequiredArgsConstructor; +import ru.di9.mirror.core.domain.ItemRecord2; +import ru.di9.mirror.core.service.MinioService; + +import java.util.List; + +@RequiredArgsConstructor +public class IndexOfHandler { + + private final MinioService minioService; + + public List walker(String path) { + return minioService.list("/local/" + path); + } + +} diff --git a/mirror-core/src/main/java/ru/di9/mirror/core/handler/MavenHandler.java b/mirror-core/src/main/java/ru/di9/mirror/core/handler/MavenHandler.java new file mode 100644 index 0000000..c6f41e8 --- /dev/null +++ b/mirror-core/src/main/java/ru/di9/mirror/core/handler/MavenHandler.java @@ -0,0 +1,50 @@ +package ru.di9.mirror.core.handler; + +import lombok.RequiredArgsConstructor; +import ru.di9.mirror.core.domain.MavenHandlerGetFileResponse; +import ru.di9.mirror.core.service.ExternalMavenService; +import ru.di9.mirror.core.service.MinioService; + +import java.io.InputStream; +import java.util.List; +import java.util.Optional; + +@RequiredArgsConstructor +public class MavenHandler { + + private final MinioService minioService; + private final List externalMavenServices; + + public Optional getFile(String path) { + final String fileName = path.substring(path.lastIndexOf("/") + 1); + + Optional optionalInputStream = minioService.get("/local/" + path); + if (optionalInputStream.isPresent()) { + return optionalInputStream + .map(inputStream -> new MavenHandlerGetFileResponse(fileName, inputStream)); + } else { + for (ExternalMavenService externalMavenService : externalMavenServices) { + final String nameForStore = "/" + externalMavenService.getId() + "/" + path; + + optionalInputStream = minioService.get(nameForStore); + if (optionalInputStream.isPresent()) { + return optionalInputStream + .map(inputStream -> new MavenHandlerGetFileResponse(fileName, inputStream)); + } else { + optionalInputStream = externalMavenService.getFile(path); + if (optionalInputStream.isPresent()) { + minioService.put(nameForStore, optionalInputStream.get()); + return minioService.get(nameForStore) + .map(inputStream -> new MavenHandlerGetFileResponse(fileName, inputStream)); + } + } + } + + return Optional.empty(); + } + } + + public void putFile(String path, InputStream inputStream) { + minioService.put("/local/" + path, inputStream); + } +} diff --git a/mirror-core/src/main/java/ru/di9/mirror/core/service/ExternalMavenService.java b/mirror-core/src/main/java/ru/di9/mirror/core/service/ExternalMavenService.java new file mode 100644 index 0000000..4eeed99 --- /dev/null +++ b/mirror-core/src/main/java/ru/di9/mirror/core/service/ExternalMavenService.java @@ -0,0 +1,55 @@ +package ru.di9.mirror.core.service; + +import lombok.Getter; +import lombok.SneakyThrows; +import lombok.ToString; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Optional; + +@SuppressWarnings("ClassCanBeRecord") +@ToString(of = {"id", "url"}) +public class ExternalMavenService { + + @Getter + private final String id; + private final String url; + private final OkHttpClient okHttpClient; + + public ExternalMavenService(String id, String url, OkHttpClient okHttpClient) { + this.id = id; + this.okHttpClient = okHttpClient; + + if (!url.endsWith("/")) { + this.url = url + "/"; + } else { + this.url = url; + } + } + + @SneakyThrows + public Optional getFile(String path) { + if (path.startsWith("/")) { + path = path.substring(1); + } + + Request request = new Request.Builder() + .url(url + path) + .build(); + + Response response = okHttpClient.newCall(request).execute(); + if (!response.isSuccessful()) { + if (response.code() == 404) { + return Optional.empty(); + } + throw new IOException("Unexpected code " + response); + } + + return Optional.ofNullable(response.body()).map(ResponseBody::byteStream); + } +} diff --git a/mirror-core/src/main/java/ru/di9/mirror/core/service/MinioService.java b/mirror-core/src/main/java/ru/di9/mirror/core/service/MinioService.java new file mode 100644 index 0000000..43a6100 --- /dev/null +++ b/mirror-core/src/main/java/ru/di9/mirror/core/service/MinioService.java @@ -0,0 +1,68 @@ +package ru.di9.mirror.core.service; + +import io.minio.*; +import io.minio.errors.*; +import io.minio.messages.Item; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import ru.di9.mirror.core.domain.ItemRecord2; + +import java.io.IOException; +import java.io.InputStream; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +@RequiredArgsConstructor +public class MinioService { + private static final long UNKNOWN_SIZE = -1L; + private static final long PART_SIZE = (5 * 1024 * 1024); // 5 MB + + private final MinioClient minioClient; + private final String bucket; + + @SneakyThrows + public void put(String name, InputStream inputStream) { + minioClient.putObject(PutObjectArgs.builder() + .bucket(bucket) + .object(name) + .stream(inputStream, UNKNOWN_SIZE, PART_SIZE) + .build()); + } + + @SneakyThrows + public Optional get(String name) { + try { + return Optional.of(minioClient.getObject(GetObjectArgs.builder() + .bucket(bucket) + .object(name) + .build())); + } catch (ErrorResponseException e) { + if (e.errorResponse().code().equalsIgnoreCase("NoSuchKey")) { + return Optional.empty(); + } else { + throw e; + } + } + } + + public List list(String prefix) { + if (!prefix.endsWith("/")) { + prefix = prefix + "/"; + } + + Iterable> results = minioClient.listObjects(ListObjectsArgs.builder() + .bucket(bucket) + .prefix(prefix) + .build()); + + List list = new ArrayList<>(); + for (Result result : results) { + list.add(new ItemRecord2(result, prefix)); + } + + return list; + } +} diff --git a/mirror-core/src/test/java/ru/di9/mirror/core/service/MinioServiceTest.java b/mirror-core/src/test/java/ru/di9/mirror/core/service/MinioServiceTest.java new file mode 100644 index 0000000..1855c57 --- /dev/null +++ b/mirror-core/src/test/java/ru/di9/mirror/core/service/MinioServiceTest.java @@ -0,0 +1,63 @@ +package ru.di9.mirror.core.service; + +import io.minio.ListObjectsArgs; +import io.minio.MinioClient; +import io.minio.Result; +import io.minio.messages.Item; +import lombok.SneakyThrows; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import ru.di9.mirror.core.domain.ItemRecord2; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +class MinioServiceTest { + static final String bucket = "mirror"; + static MinioClient mockMinioClient; + + @BeforeAll + static void beforeAll() { + mockMinioClient = mock(MinioClient.class); + + Iterable> listObjects = List.of( + mockResult("ghast/ghast-tools/maven-metadata.xml", 320, false), + mockResult("ghast/ghast-tools/1.13/", 0, true) + ); + when(mockMinioClient.listObjects(any(ListObjectsArgs.class))).thenReturn(listObjects); + } + + @Test + void list() { + MinioService minioService = new MinioService(mockMinioClient, bucket); + List list = minioService.list("/ghast/ghast-tools/"); + + assertNotNull(list); + assertFalse(list.isEmpty()); + + assertTrue(list.stream() + .anyMatch(itemRecord2 -> itemRecord2.name().equals("maven-metadata.xml") + && !itemRecord2.isDir() + && itemRecord2.size() == 320)); + assertTrue(list.stream() + .anyMatch(itemRecord2 -> itemRecord2.name().equals("1.13") + && itemRecord2.isDir() + && itemRecord2.size() == 0)); + } + + @SuppressWarnings("unchecked") + @SneakyThrows + private static Result mockResult(String name, long size, boolean isDir) { + Item item = mock(Item.class); + when(item.objectName()).thenReturn(name); + when(item.size()).thenReturn(size); + when(item.isDir()).thenReturn(isDir); + + Result result = mock(Result.class); + when(result.get()).thenReturn(item); + + return result; + } +} diff --git a/mirror-web/build.gradle b/mirror-web/build.gradle index 8e6187c..20ba1fd 100644 --- a/mirror-web/build.gradle +++ b/mirror-web/build.gradle @@ -1,11 +1,12 @@ +//file:noinspection GrUnresolvedAccess apply plugin: 'org.springframework.boot' -apply plugin: 'io.spring.dependency-management' dependencies { - implementation(project(':mirror-app')) + implementation(project(':mirror-core')) - annotationProcessor('org.springframework.boot:spring-boot-configuration-processor') - implementation('org.springframework.boot:spring-boot-starter-web') + annotationProcessor("org.springframework.boot:spring-boot-configuration-processor:${springBootVerson}") + implementation("org.springframework.boot:spring-boot-starter-web:${springBootVerson}") + implementation("org.springframework.boot:spring-boot-starter-freemarker:${springBootVerson}") } tasks.named('compileJava') { diff --git a/mirror-web/src/main/java/ru/di9/mirror/web/config/MavenMirrorsProperties.java b/mirror-web/src/main/java/ru/di9/mirror/web/config/MavenMirrorsProperties.java new file mode 100644 index 0000000..20aa727 --- /dev/null +++ b/mirror-web/src/main/java/ru/di9/mirror/web/config/MavenMirrorsProperties.java @@ -0,0 +1,20 @@ +package ru.di9.mirror.web.config; + +import lombok.Getter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; + +@Component +@ConfigurationProperties(prefix = "maven-mirrors") +public class MavenMirrorsProperties { + + @Getter + private final List list = new ArrayList<>(); + + public record MirrorData(String id, String url) { + + } +} diff --git a/mirror-web/src/main/java/ru/di9/mirror/web/config/WebConfig.java b/mirror-web/src/main/java/ru/di9/mirror/web/config/WebConfig.java index 44389a8..1d504e1 100644 --- a/mirror-web/src/main/java/ru/di9/mirror/web/config/WebConfig.java +++ b/mirror-web/src/main/java/ru/di9/mirror/web/config/WebConfig.java @@ -1,10 +1,14 @@ package ru.di9.mirror.web.config; +import io.minio.MinioClient; import okhttp3.OkHttpClient; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import ru.di9.mirror.app.service.*; +import ru.di9.mirror.core.handler.IndexOfHandler; +import ru.di9.mirror.core.handler.MavenHandler; +import ru.di9.mirror.core.service.ExternalMavenService; +import ru.di9.mirror.core.service.MinioService; import java.util.List; @@ -12,20 +16,42 @@ import java.util.List; public class WebConfig { @Bean - public StorageService mirroredStorageService(FileStorageService fileStorageService, - List mirrors) { - return new MirroredStorageService(fileStorageService, mirrors); + public MavenHandler mavenHandler(MinioService minioService, + List externalMavenServices) { + return new MavenHandler(minioService, externalMavenServices); } @Bean - public FileStorageService fileStorageService(@Value("${maven.storage}") String storagePath) { - return new FileStorageService(storagePath); + public IndexOfHandler indexOfHandler(MinioService minioService) { + return new IndexOfHandler(minioService); } @Bean - public List mirrors() { - return List.of( - new MavenCentralStorageService(new OkHttpClient()) - ); + public MinioService minioService(MinioClient minioClient, @Value("${minio.bucket}") String bucket) { + return new MinioService(minioClient, bucket); } + + @Bean + public MinioClient minioClient(@Value("${minio.url}") String url, + @Value("${minio.accessKey}") String accessKey, + @Value("${minio.secretKey}") String secretKey) { + return MinioClient.builder() + .endpoint(url) + .credentials(accessKey, secretKey) + .build(); + } + + @Bean + public List externalMavenServices(OkHttpClient okHttpClient, + MavenMirrorsProperties mavenMirrorsProperties) { + return mavenMirrorsProperties.getList().stream() + .map(mirrorData -> new ExternalMavenService(mirrorData.id(), mirrorData.url(), okHttpClient)) + .toList(); + } + + @Bean + public OkHttpClient okHttpClient() { + return new OkHttpClient(); + } + } diff --git a/mirror-web/src/main/java/ru/di9/mirror/web/controller/IndexOfMavenController.java b/mirror-web/src/main/java/ru/di9/mirror/web/controller/IndexOfMavenController.java index 12946de..056ea51 100644 --- a/mirror-web/src/main/java/ru/di9/mirror/web/controller/IndexOfMavenController.java +++ b/mirror-web/src/main/java/ru/di9/mirror/web/controller/IndexOfMavenController.java @@ -1,86 +1,79 @@ package ru.di9.mirror.web.controller; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.core.io.InputStreamResource; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Controller; +import org.springframework.ui.ModelMap; import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.ModelAttribute; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; -import ru.di9.mirror.app.Utils; -import ru.di9.mirror.app.domain.FileRecord; -import ru.di9.mirror.app.domain.PathRecord; -import ru.di9.mirror.app.domain.PathType; -import ru.di9.mirror.app.domain.WalkerResult; -import ru.di9.mirror.app.service.StorageService; +import ru.di9.mirror.core.Utils; +import ru.di9.mirror.core.domain.ItemRecord2; +import ru.di9.mirror.core.handler.IndexOfHandler; +import java.util.ArrayList; import java.util.List; -import java.util.Optional; @Controller @RequestMapping(path = "/maven") public class IndexOfMavenController { - private final StorageService storageService; - public IndexOfMavenController(@Qualifier("mirroredStorageService") StorageService storageService) { - this.storageService = storageService; + private final IndexOfHandler indexOfHandler; + + public IndexOfMavenController(IndexOfHandler indexOfHandler) { + this.indexOfHandler = indexOfHandler; } @GetMapping(path = "/{*path}", headers = "accept=text/html") - public ResponseEntity walker(@PathVariable("path") String httpPath) { - ResponseEntity responseEntity; + public String walker(@PathVariable("path") String httpPath, + @ModelAttribute("model") ModelMap model) { + String path = correctingHttpPath(httpPath); - Optional walkerResultOptional = storageService.walker(httpPath); - if (walkerResultOptional.isEmpty()) { - responseEntity = ResponseEntity.status(HttpStatus.NOT_FOUND) - .body("

404 - Not Found

Project-Mirror

"); - } else { - WalkerResult walkerResult = walkerResultOptional.get(); - if (walkerResult.pathType() == PathType.DIRECTORY) { - responseEntity = generateIndexOf(httpPath, walkerResult.filesList()); - } else { - responseEntity = downloadFile(walkerResult.fileRecord()); + List walker = indexOfHandler.walker(path); + List links = new ArrayList<>(); + + StringBuilder sb = new StringBuilder("/maven"); + int resetLength = sb.length(); + + if (Utils.isNotEmptyString(path)) { + int idx = path.lastIndexOf("/"); + if (idx > -1) { + sb.append("/").append(path, 0, idx); } + + links.add(new ModelLink("..", sb.toString())); } - return responseEntity; - } + sb.setLength(resetLength); + sb.append("/"); + resetLength = sb.length(); + for (ItemRecord2 itemRecord2 : walker) { + if (Utils.isNotEmptyString(path)) { + sb.append(path).append("/"); + } + sb.append(itemRecord2.name()); - private ResponseEntity downloadFile(FileRecord fileRecord) { - return ResponseEntity.ok() - .header(HttpHeaders.CONTENT_DISPOSITION, - "attachment; filename=\"" + fileRecord.name() + "\"") - .contentType(MediaType.APPLICATION_OCTET_STREAM) - .contentLength(fileRecord.size()) - .body(new InputStreamResource(fileRecord.inputStream())); - } - - private ResponseEntity generateIndexOf(String httpPath, List filesList) { - ResponseEntity responseEntity; - StringBuilder sb = new StringBuilder("

Index of

Project-Mirror

").toString()); - return responseEntity; + private String correctingHttpPath(String httpPath) { + String path = httpPath; + + if (path.startsWith("/")) { + path = path.substring(1); + } + + if (path.endsWith("/")) { + path = path.substring(0, path.length() - 1); + } + + return path; + } + + public record ModelLink(String name, String link) { } } diff --git a/mirror-web/src/main/java/ru/di9/mirror/web/controller/MavenController.java b/mirror-web/src/main/java/ru/di9/mirror/web/controller/MavenController.java index eef5e8c..a089167 100644 --- a/mirror-web/src/main/java/ru/di9/mirror/web/controller/MavenController.java +++ b/mirror-web/src/main/java/ru/di9/mirror/web/controller/MavenController.java @@ -1,7 +1,6 @@ package ru.di9.mirror.web.controller; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.core.io.InputStreamResource; import org.springframework.core.io.Resource; import org.springframework.http.HttpHeaders; @@ -12,43 +11,31 @@ import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestMapping; -import ru.di9.mirror.app.domain.FileRecord; -import ru.di9.mirror.app.service.StorageService; +import ru.di9.mirror.core.handler.MavenHandler; import javax.servlet.http.HttpServletRequest; import java.io.IOException; import java.io.InputStream; -import java.util.Optional; @Slf4j @Controller @RequestMapping(path = "/maven") public class MavenController { - private final StorageService storageService; + private final MavenHandler mavenHandler; - public MavenController(@Qualifier("mirroredStorageService") StorageService storageService) { - this.storageService = storageService; + public MavenController(MavenHandler mavenHandler) { + this.mavenHandler = mavenHandler; } - @GetMapping(path = "/{*path}") + @GetMapping(path = "/{*path}", produces = MediaType.APPLICATION_OCTET_STREAM_VALUE) public ResponseEntity mavenGetFile(@PathVariable("path") String httpPath) { - ResponseEntity responseEntity; - - Optional fileRecordOptional = storageService.getFile(httpPath); - if (fileRecordOptional.isEmpty()) { - responseEntity = ResponseEntity.notFound().build(); - } else { - FileRecord fileRecord = fileRecordOptional.get(); - - responseEntity = ResponseEntity.ok() + return mavenHandler.getFile(httpPath.substring(1)) + .>map(response -> ResponseEntity.ok() .header(HttpHeaders.CONTENT_DISPOSITION, - "attachment; filename=\"" + fileRecord.name() + "\"") + "attachment; filename=\"" + response.name() + "\"") .contentType(MediaType.APPLICATION_OCTET_STREAM) - .contentLength(fileRecord.size()) - .body(new InputStreamResource(fileRecord.inputStream())); - } - - return responseEntity; + .body(new InputStreamResource(response.inputStream()))) + .orElseGet(() -> ResponseEntity.notFound().build()); } @SuppressWarnings("java:S1452") @@ -57,7 +44,7 @@ public class MavenController { HttpServletRequest httpRequest) throws IOException { try (InputStream inputStream = httpRequest.getInputStream()) { - storageService.putFile(httpPath, inputStream); + mavenHandler.putFile(httpPath.substring(1), inputStream); } return ResponseEntity.noContent().build(); diff --git a/mirror-web/src/main/resources/META-INF/additional-spring-configuration-metadata.json b/mirror-web/src/main/resources/META-INF/additional-spring-configuration-metadata.json index 8f55c4b..5f8640c 100644 --- a/mirror-web/src/main/resources/META-INF/additional-spring-configuration-metadata.json +++ b/mirror-web/src/main/resources/META-INF/additional-spring-configuration-metadata.json @@ -1,8 +1,28 @@ { - "properties": [ - { - "name": "maven.storage", - "type": "java.lang.String" + "properties": [ + { + "name": "minio.url", + "type": "java.lang.String", + "description": "MinIO console url." + }, + { + "name": "minio.accessKey", + "type": "java.lang.String", + "description": "MinIO Access key (login)." + }, + { + "name": "minio.secretKey", + "type": "java.lang.String", + "description": "MinIO Secret key (password)." + }, + { + "name": "minio.bucket", + "type": "java.lang.String", + "description": "MinIO Bucket name." + }, + { + "name": "maven-mirrors.list", + "type": "java.util.List", + "sourceType": "java.util.List" } - ] -} \ No newline at end of file +] } diff --git a/mirror-web/src/main/resources/application.yml b/mirror-web/src/main/resources/application.yml index 18e838d..0b74f11 100644 --- a/mirror-web/src/main/resources/application.yml +++ b/mirror-web/src/main/resources/application.yml @@ -2,7 +2,22 @@ server: address: 127.0.0.1 port: 8080 +spring: + freemarker: + template-loader-path: classpath:/templates + suffix: .ftl + debug: false -maven: - storage: './storage' +minio: + url: 'http://dev.di9.ru:9000' + accessKey: 'mirror' + secretKey: 'mirror123' + bucket: 'mirror' + +maven-mirrors: + list: + - id: 'maven_central' + url: 'https://repo1.maven.org/maven2' + - id: 'spigot' + url: 'https://hub.spigotmc.org/nexus/content/groups/public' diff --git a/mirror-web/src/main/resources/templates/list.ftl b/mirror-web/src/main/resources/templates/list.ftl new file mode 100644 index 0000000..3f77fdf --- /dev/null +++ b/mirror-web/src/main/resources/templates/list.ftl @@ -0,0 +1,19 @@ + + +
+ + Project-Mirror +
+ +

Index of

+
+
    + <#--noinspection FtlTypesInspection--> + <#list model["links"] as item> +
  • ${item.name()}
    • + +
+
+

Project-Mirror

+ + diff --git a/settings.gradle b/settings.gradle index 0cfa533..4b20231 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,4 +1,4 @@ rootProject.name = 'project-mirror' -include('mirror-app') +include('mirror-core') include('mirror-web') diff --git a/versions.gradle b/versions.gradle index 385d46c..a425c99 100644 --- a/versions.gradle +++ b/versions.gradle @@ -1,7 +1,9 @@ ext { lombokVersion = '1.18.20' - okhttpVersion = '4.9.3' + minioVersion = '8.3.9' + springBootVerson = '2.6.6' - pluginSpringBootVerson = '2.6.6' - pluginSpringBomVerson = '1.0.11.RELEASE' + // for tests only + junitJupiterVersion = '5.5.2' + mockitoCoreVersion = '4.5.1' }