diff --git a/.editorconfig b/.editorconfig index dc312cc..20a38a0 100644 --- a/.editorconfig +++ b/.editorconfig @@ -6,10 +6,3 @@ end_of_line = lf indent_size = 4 indent_style = space insert_final_newline = true - -[gradlew.bat] -charset = latin1 -end_of_line = crlf - -[*.{yml, yaml, json}] -indent_size = 2 diff --git a/.gitignore b/.gitignore index d1dae46..41b9db0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,10 @@ -# IDEA # +# JetBrains IDEA .idea/ *.iml -# GRADLE # +# Gradle .gradle/ build/ -# OTHER # +# Application storage/ diff --git a/README.MD b/README.MD index 525d17c..ae2a08e 100644 --- a/README.MD +++ b/README.MD @@ -1,20 +1,77 @@ -# Project "MIRROR" -Зеркалирующий мавен-репозиторий. +# PROJECT "MIRROR" -## Сборка -```shell -gradle :web-spring:bootDistTar +Зеркалирование артефактов Maven репозиториев. + +## Как подключать "MIRROR" к сборщикам + +### Gradle + +```groovy +repositories { + maven { + url = "http://example.com:8080/maven/central" + } +} ``` -или +Где `central` в URL - идентификатор зеркалируемого репозитория, указанного в файле настроек. -```shell -gradle :web-spring:installBootDist +### Maven + +```xml + + + mirror-central + http://example.com:8080/maven/central + + ``` -## Запуск +Где `central` в URL - идентификатор зеркалируемого репозитория, указанного в файле настроек. + +## Запуск и настройка + +### Требования к запуску + +- Java 17 + +### Настройка + +_Пример настроек можно посмотреть в файле [application.properties](src/main/resources/application.properties)._ + +Создайте файл `config.properties` и укажите в нём следующие настройки + +#### Общие настройки + +| Настройка | Описание | Значение по-умолчанию | +|---------------------|-------------------------------------------------------------------|-----------------------| +| `server.addres` | Интерфейс, который будет прослушиваться | `0.0.0.0` | +| `server.port` | Порт, который будет прослушиваться | `8080` | +| `app.maven.storage` | Путь к папке, в которую будут сохраняться отзеркаленные артефакты | `storage` | + +#### Настройки репозиториев + +Значение `[0]` в настройках указывает на порядковый номер. Сам порядок ни на что не влияет, однако каждая следующая +настройка репозитория должна увеличивать это значение на единицу: `[0]`, `[1]`, `[2]` и так далее. + +| Настройка | Описание | +|--------------------------------------|------------------------------------------------------------------------| +| `app.maven.repository[0].id` | Уникальный идентификатор репозитория | +| `app.maven.repository[0].url` | URL репозитория | +| `app.maven.repository[0].cache-time` | Сколько времени хранить информацию о не найденных ресурсах (в минутах) | + +### Запуск + ```shell -./bin/web-spring +bin/project-mirror --spring.config.location=path/to/config.properties ``` -Все настройки производятся через стандартный "спринговский" `application.yml`. +## Сборка из исходников + +```shell +gradle installBootDist +``` + +В директории `build/install/project-mirror-boot` будут находиться +скрипты запуска (в `bin`) +и само приложение (в `lib`). diff --git a/build.gradle b/build.gradle index 2506fa4..7720770 100644 --- a/build.gradle +++ b/build.gradle @@ -1,34 +1,45 @@ -//file:noinspection GrUnresolvedAccess -buildscript { - apply from: 'dependencies.gradle' - - repositories { - maven { url 'https://plugins.gradle.org/m2/' } - } - dependencies { - // https://plugins.gradle.org/plugin/org.springframework.boot - classpath deps.springboot.plugin - } +plugins { + id("java") + id("application") + id("org.springframework.boot") version("3.4.0") + id("io.spring.dependency-management") version("1.1.6") } -subprojects { - apply plugin: 'java' - - group = 'ru.di9.mirror' - version = '1.0' - - compileJava { - targetCompatibility = sourceCompatibility = JavaVersion.VERSION_17 - options.encoding = 'UTF-8' - } - - repositories { - mavenCentral() - } - - dependencies { - annotationProcessor(deps.lombok) - compileOnly(deps.lombok) - implementation(deps.logger.api) - } +wrapper { + gradleVersion = "8.10" + distributionType = Wrapper.DistributionType.BIN +} + +compileJava { + sourceCompatibility = targetCompatibility = JavaVersion.VERSION_17 + options.encoding = "UTF-8" +} + +group = "ru.di9.mirror" +version = "1.1-SNAPSHOT" + +repositories { + mavenLocal() + mavenCentral() +} + +ext { + lombokVersion = "1.18.34" +} + +dependencies { + // Lombok + annotationProcessor("org.projectlombok:lombok:$lombokVersion") + compileOnly("org.projectlombok:lombok:$lombokVersion") + + // Spring Boot Web + implementation("org.springframework.boot:spring-boot-starter-web") + + // Other + implementation("org.apache.httpcomponents.client5:httpclient5:5.5") + implementation("commons-io:commons-io:2.14.0") +} + +application { + mainClass = "ru.di9.mirror.Application" } diff --git a/core/src/main/java/ru/di9/mirror/core/FileRecord.java b/core/src/main/java/ru/di9/mirror/core/FileRecord.java deleted file mode 100644 index 4881ca8..0000000 --- a/core/src/main/java/ru/di9/mirror/core/FileRecord.java +++ /dev/null @@ -1,6 +0,0 @@ -package ru.di9.mirror.core; - -import java.io.InputStream; - -public record FileRecord(String name, long size, InputStream inputStream) { -} diff --git a/core/src/main/java/ru/di9/mirror/core/FileStorage.java b/core/src/main/java/ru/di9/mirror/core/FileStorage.java deleted file mode 100644 index bdbe8a3..0000000 --- a/core/src/main/java/ru/di9/mirror/core/FileStorage.java +++ /dev/null @@ -1,62 +0,0 @@ -package ru.di9.mirror.core; - -import lombok.Getter; -import lombok.SneakyThrows; - -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Optional; - -import static ru.di9.mirror.core.Utils.isEmptyString; - -@SuppressWarnings("ClassCanBeRecord") -public class FileStorage { - - @Getter - private final String id; - private final Path storagePath; - - public FileStorage(String id, Path storagePath) { - this.id = id; - this.storagePath = storagePath.resolve(this.id); - } - - @SneakyThrows - public Optional getFile(String path) { - Path fullPath = storageResolve(path); - - if (Files.notExists(fullPath)) { - return Optional.empty(); - } - - return Optional.of(new FileRecord( - fullPath.toFile().getName(), - Files.size(fullPath), - Files.newInputStream(fullPath) - )); - } - - @SneakyThrows - public void putFile(String path, InputStream inputStream) { - Path fullPath = storageResolve(path); - Files.createDirectories(fullPath.getParent()); - - try (OutputStream outputStream = Files.newOutputStream(fullPath)) { - byte[] buff = new byte[8 * 1024]; - int len; - while ((len = inputStream.read(buff)) > 0) { - outputStream.write(buff, 0, len); - } - } - } - - private Path storageResolve(String httpPath) { - if (isEmptyString(httpPath)) { - return storagePath; - } else { - return storagePath.resolve(httpPath.substring(1)); - } - } -} diff --git a/core/src/main/java/ru/di9/mirror/core/MavenClient.java b/core/src/main/java/ru/di9/mirror/core/MavenClient.java deleted file mode 100644 index 43851e4..0000000 --- a/core/src/main/java/ru/di9/mirror/core/MavenClient.java +++ /dev/null @@ -1,47 +0,0 @@ -package ru.di9.mirror.core; - -import lombok.SneakyThrows; - -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.time.Duration; -import java.util.Optional; - -public class MavenClient { - - private final String url; - private final HttpClient httpClient; - - public MavenClient(String url) { - this.url = url + (url.endsWith("/") ? url.substring(0, url.length() - 1) : ""); - - this.httpClient = HttpClient.newBuilder() - .version(HttpClient.Version.HTTP_1_1) - .connectTimeout(Duration.ofSeconds(10)) - .build(); - } - - @SneakyThrows - public Optional getFile(String path) { - HttpRequest request = HttpRequest.newBuilder() - .GET().uri(URI.create(url + path)) - .build(); - - HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream()); - if (response.statusCode() == 404) { - return Optional.empty(); - } else if (response.statusCode() == 200) { - return Optional.of(new FileRecord( - path, - response.headers().firstValueAsLong("Content-Length").orElse(-1L), - response.body() - )); - } else { - throw new IOException("Unexpected code " + response.statusCode()); - } - } -} diff --git a/core/src/main/java/ru/di9/mirror/core/MirrorService.java b/core/src/main/java/ru/di9/mirror/core/MirrorService.java deleted file mode 100644 index 382cdde..0000000 --- a/core/src/main/java/ru/di9/mirror/core/MirrorService.java +++ /dev/null @@ -1,36 +0,0 @@ -package ru.di9.mirror.core; - -import lombok.RequiredArgsConstructor; - -import java.util.List; -import java.util.Optional; - -@RequiredArgsConstructor -public class MirrorService { - - private final List mavenList; - - public Optional getFile(String path) { - Optional optional; - - for (MirrorPair mirrorPair : mavenList) { - optional = mirrorPair.fileStorage().getFile(path); - if (optional.isPresent()) { - return optional; - } - } - - for (MirrorPair mirrorPair : mavenList) { - optional = mirrorPair.mavenClient().getFile(path); - if (optional.isPresent()) { - mirrorPair.fileStorage().putFile(path, optional.get().inputStream()); - return mirrorPair.fileStorage().getFile(path); - } - } - - return Optional.empty(); - } - - public record MirrorPair(FileStorage fileStorage, MavenClient mavenClient) { - } -} diff --git a/core/src/main/java/ru/di9/mirror/core/Utils.java b/core/src/main/java/ru/di9/mirror/core/Utils.java deleted file mode 100644 index 074335e..0000000 --- a/core/src/main/java/ru/di9/mirror/core/Utils.java +++ /dev/null @@ -1,12 +0,0 @@ -package ru.di9.mirror.core; - -import lombok.AccessLevel; -import lombok.NoArgsConstructor; - -@NoArgsConstructor(access = AccessLevel.PRIVATE) -public class Utils { - - public static boolean isEmptyString(String string) { - return string == null || string.isEmpty() || string.isBlank(); - } -} diff --git a/dependencies.gradle b/dependencies.gradle deleted file mode 100644 index f173f4b..0000000 --- a/dependencies.gradle +++ /dev/null @@ -1,19 +0,0 @@ -//file:noinspection SpellCheckingInspection -def springboot_version = '2.6.6' - -ext { - deps = [ - lombok: 'org.projectlombok:lombok:1.18.20' - ] - - deps.logger = [ - api: 'org.slf4j:slf4j-api:1.7.30' - ] - - deps.springboot = [ - // https://plugins.gradle.org/plugin/org.springframework.boot - plugin: "org.springframework.boot:spring-boot-gradle-plugin:${springboot_version}", - web: "org.springframework.boot:spring-boot-starter-web:${springboot_version}", - config_processor: "org.springframework.boot:spring-boot-configuration-processor:${springboot_version}" - ] -} diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 41d9927..a4b76b9 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 2ec77e5..9355b41 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.5-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-bin.zip +networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index 1b6c787..f5feea6 100755 --- a/gradlew +++ b/gradlew @@ -15,6 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# SPDX-License-Identifier: Apache-2.0 +# ############################################################################## # @@ -55,7 +57,7 @@ # Darwin, MinGW, and NonStop. # # (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt # within the Gradle project. # # You can find Gradle at https://github.com/gradle/gradle/. @@ -80,13 +82,12 @@ do esac done -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit - -APP_NAME="Gradle" +# This is normally unused +# shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s +' "$PWD" ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -133,22 +134,29 @@ location of your Java installation." fi else JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac case $MAX_FD in #( '' | soft) :;; #( *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -193,11 +201,15 @@ if "$cygwin" || "$msys" ; then done fi -# Collect all arguments for the java command; -# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of -# shell script including quotes and variable substitutions, so put them in -# double quotes to make sure that they get re-expanded; and -# * put everything else in single quotes, so that it's not re-expanded. + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ @@ -205,6 +217,12 @@ set -- \ org.gradle.wrapper.GradleWrapperMain \ "$@" +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + # Use "xargs" to parse quoted args. # # With -n1 it outputs one arg per line, with the quotes and backslashes removed. diff --git a/gradlew.bat b/gradlew.bat index ac1b06f..9b42019 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -13,8 +13,10 @@ @rem See the License for the specific language governing permissions and @rem limitations under the License. @rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem -@if "%DEBUG%" == "" @echo off +@if "%DEBUG%"=="" @echo off @rem ########################################################################## @rem @rem Gradle startup script for Windows @@ -25,7 +27,8 @@ if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @@ -40,13 +43,13 @@ if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto execute +if %ERRORLEVEL% equ 0 goto execute -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail @@ -56,11 +59,11 @@ set JAVA_EXE=%JAVA_HOME%/bin/java.exe if exist "%JAVA_EXE%" goto execute -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail @@ -75,13 +78,15 @@ set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar :end @rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd +if %ERRORLEVEL% equ 0 goto mainEnd :fail rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% :mainEnd if "%OS%"=="Windows_NT" endlocal diff --git a/settings.gradle b/settings.gradle index 68a7487..896271f 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,6 +1 @@ -rootProject.name = 'project-mirror' - -include( - 'core', - 'web-spring' -) \ No newline at end of file +rootProject.name = "project-mirror" diff --git a/web-spring/src/main/java/ru/di9/mirror/web/Application.java b/src/main/java/ru/di9/mirror/Application.java similarity index 52% rename from web-spring/src/main/java/ru/di9/mirror/web/Application.java rename to src/main/java/ru/di9/mirror/Application.java index fd6a6b2..e3751f5 100644 --- a/web-spring/src/main/java/ru/di9/mirror/web/Application.java +++ b/src/main/java/ru/di9/mirror/Application.java @@ -1,11 +1,19 @@ -package ru.di9.mirror.web; +package ru.di9.mirror; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +/** + * Класс запуска приложения. + */ @SpringBootApplication public class Application { + /** + * Точка запуска приложения. + * + * @param args переданные аргументы командной строки + */ public static void main(String[] args) { SpringApplication.run(Application.class, args); } diff --git a/src/main/java/ru/di9/mirror/config/MavenMirrorRepositoryProperties.java b/src/main/java/ru/di9/mirror/config/MavenMirrorRepositoryProperties.java new file mode 100644 index 0000000..1096734 --- /dev/null +++ b/src/main/java/ru/di9/mirror/config/MavenMirrorRepositoryProperties.java @@ -0,0 +1,42 @@ +package ru.di9.mirror.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import java.util.List; + +/** + * Класс хранения конфигурации приложения. + */ +@Component +@ConfigurationProperties(prefix = "app.maven") +@Getter +@Setter +public class MavenMirrorRepositoryProperties { + /** + * Путь до хранения ресурсов в файловой системе. + */ + private String storage; + + /** + * Список настроек удалённых репозиториев. + */ + private List repository; + + /** + * Настройки удалённого репозитория. + * + * @param id уникальный идентификатор репозитория + * @param url URL удалённого репозитория + * @param cacheTime время актуальности кэша не найденных ресурсов + */ + public record MavenMirrorRepositoryData(String id, String url, Integer cacheTime) { + public MavenMirrorRepositoryData(String id, String url, Integer cacheTime) { + this.id = id; + this.url = url; + this.cacheTime = (cacheTime == null ? 1 : cacheTime); + } + } +} diff --git a/src/main/java/ru/di9/mirror/controller/MavenController.java b/src/main/java/ru/di9/mirror/controller/MavenController.java new file mode 100644 index 0000000..9bad648 --- /dev/null +++ b/src/main/java/ru/di9/mirror/controller/MavenController.java @@ -0,0 +1,63 @@ +package ru.di9.mirror.controller; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.core.io.InputStreamResource; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import ru.di9.mirror.service.MavenService; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import java.util.Optional; + +/** + * Контроллер Maven-like репозитория. + */ +@Controller +@RequestMapping(path = "/maven") +@RequiredArgsConstructor +@Slf4j +public class MavenController { + private final MavenService mavenService; + + /** + * Запрос ресурса. + * + * @param id идентификатор репозитория + * @param resourcePath путь к ресурсу + * @return ресурс, если такой существует + */ + @GetMapping(path = "/{id}/{*path}", produces = { + MediaType.APPLICATION_OCTET_STREAM_VALUE, + MediaType.TEXT_PLAIN_VALUE + }) + public ResponseEntity resourceGet( + @PathVariable("id") String id, + @PathVariable("path") String resourcePath + ) { + log.debug("GET '{}'", resourcePath); + Optional optResource = mavenService.getResource(id, resourcePath); + if (optResource.isEmpty()) { + return ResponseEntity.notFound().build(); + } + + File file = optResource.get(); + try { + return ResponseEntity.ok() + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + file.getName() + "\"") + .contentType(MediaType.APPLICATION_OCTET_STREAM) + .contentLength(file.length()) + .body(new InputStreamResource(Files.newInputStream(file.toPath(), StandardOpenOption.READ))); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/main/java/ru/di9/mirror/domain/FileStorage.java b/src/main/java/ru/di9/mirror/domain/FileStorage.java new file mode 100644 index 0000000..95ebe90 --- /dev/null +++ b/src/main/java/ru/di9/mirror/domain/FileStorage.java @@ -0,0 +1,63 @@ +package ru.di9.mirror.domain; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.Optional; + +/** + * Класс работы с файловым хранилищем. + */ +public class FileStorage { + private final Path storagePath; + + /** + * Создание класса. + * + * @param storagePath путь к папке, куда сохраняем файлы + * @throws IOException проблемы с файловой системой + */ + public FileStorage(String storagePath) throws IOException { + this.storagePath = Path.of(storagePath); + if (Files.notExists(this.storagePath)) { + Files.createDirectories(this.storagePath); + } + } + + /** + * Получить файл. + * + * @param path путь к файлу, относительно месту хранения + * @return файл, если таковой найден + */ + public Optional getFile(String path) { + Path fullPath = storagePath.resolve(path.startsWith("/") ? path.substring(1) : path); + + if (Files.notExists(fullPath)) { + return Optional.empty(); + } + + return Optional.of(fullPath.toFile()); + } + + /** + * Сохранение файла в хранилище, путём перемещения его из другого места в файловой системе. + * + * @param path путь сохранения файла, относительно места хранения + * @param source откуда будет файл перемещён + */ + public void putFile(String path, Path source) { + Path fullPath = storagePath.resolve(path.startsWith("/") ? path.substring(1) : path); + try { + var parent = fullPath.getParent(); + if (Files.notExists(parent)) { + Files.createDirectories(parent); + } + Files.move(source, fullPath, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/main/java/ru/di9/mirror/domain/MavenMirrorRepository.java b/src/main/java/ru/di9/mirror/domain/MavenMirrorRepository.java new file mode 100644 index 0000000..fc79fb1 --- /dev/null +++ b/src/main/java/ru/di9/mirror/domain/MavenMirrorRepository.java @@ -0,0 +1,85 @@ +package ru.di9.mirror.domain; + +import lombok.extern.slf4j.Slf4j; +import ru.di9.mirror.utils.Cache404; +import ru.di9.mirror.utils.MavenClient; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Optional; + +/** + * Класс работы с удалённым репозиторием. + */ +@Slf4j +public class MavenMirrorRepository { + private final MavenClient mavenClient; + private final FileStorage fileStorage; + private final Cache404 cache404; + + /** + * Создание класса. + * + * @param mavenUrl базовый URL удалённого репозитория + * @param storagePath путь хранения ресурсов в файловой системе + * @param cacheTime актуальность кэша не найденных удалённых ресурсов + * @throws IOException проблемы с файловым хранилищем + */ + public MavenMirrorRepository(String mavenUrl, String storagePath, int cacheTime) throws IOException { + this.mavenClient = new MavenClient(mavenUrl); + this.fileStorage = new FileStorage(storagePath); + this.cache404 = new Cache404(cacheTime); + } + + /** + * Получить ресурс. + * + * @param path путь к ресурсу + * @return ресурс, если таковой найден + */ + public Optional getResource(String path) { + Optional optFile = fileStorage.getFile(path); + if (optFile.isPresent()) { + log.debug("Get from storage '{}'", path); + return optFile; + } + + if (cache404.contains(path)) { + log.debug("Cached 404 for '{}'", path); + return Optional.empty(); + } + + Path tempFile = null; + try { + tempFile = Files.createTempFile("prj_mir_", ".tmp"); + try (var out = Files.newOutputStream(tempFile, StandardOpenOption.WRITE)) { + if (!mavenClient.getResource(path, out)) { + log.debug("Put 404 cache for '{}'", path); + cache404.put(path); + return Optional.empty(); + } + } + log.debug("Save to storage '{}'", path); + fileStorage.putFile(path, tempFile); + optFile = fileStorage.getFile(path); + if (optFile.isEmpty()) { + log.error("Cant find saved file '{}'", path); + } + return optFile; + } catch (IOException e) { + log.error("Cant load resource: {}", e.getMessage(), e); + return Optional.empty(); + } finally { + if (tempFile != null && Files.exists(tempFile)) { + try { + Files.delete(tempFile); + } catch (IOException e) { + log.error("Cant remove temp file '{}'", tempFile, e); + } + } + } + } +} diff --git a/src/main/java/ru/di9/mirror/service/MavenService.java b/src/main/java/ru/di9/mirror/service/MavenService.java new file mode 100644 index 0000000..ccc138b --- /dev/null +++ b/src/main/java/ru/di9/mirror/service/MavenService.java @@ -0,0 +1,52 @@ +package ru.di9.mirror.service; + +import org.springframework.stereotype.Service; +import ru.di9.mirror.config.MavenMirrorRepositoryProperties; +import ru.di9.mirror.config.MavenMirrorRepositoryProperties.MavenMirrorRepositoryData; +import ru.di9.mirror.domain.MavenMirrorRepository; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +/** + * Сервис обработки запроса получения ресурса. + */ +@Service +public class MavenService { + private final Map map = new HashMap<>(); + + /** + * Создание сервиса. + * + * @param props настройки + * @throws IOException ошибка при работе с файловым хранилищем + */ + public MavenService(MavenMirrorRepositoryProperties props) throws IOException { + for (MavenMirrorRepositoryData it : props.getRepository()) { + map.put(it.id(), new MavenMirrorRepository( + it.url(), + props.getStorage() + File.separatorChar + it.id(), + it.cacheTime())); + } + } + + /** + * Получить ресурс из удалённого репозитория. + * + * @param id идентификатор репозитория + * @param path путь к ресурсу + * @return ресурс, если таковой существует + */ + public Optional getResource(String id, String path) { + MavenMirrorRepository repository = map.get(id); + if (repository == null) { + //думаю лучше кидать ошибку о несуществующем id + return Optional.empty(); + } + + return repository.getResource(path); + } +} diff --git a/src/main/java/ru/di9/mirror/utils/Cache404.java b/src/main/java/ru/di9/mirror/utils/Cache404.java new file mode 100644 index 0000000..678faf6 --- /dev/null +++ b/src/main/java/ru/di9/mirror/utils/Cache404.java @@ -0,0 +1,76 @@ +package ru.di9.mirror.utils; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * Класс кэширования не найденных (404) удалённых ресурсов. + *

+ * Необходим для того, чтобы не "задалбывать" удалённые репозитории запросами ресурсов, которых у них нет. + * Т.е. мы один раз опрашиваем удалённый репозиторий на получение ресурса. И если ресурса нет, то сохраняем + * эту информацию в этот класс. Если в течении указанного времени актуальности кэша будет повторный запрос ресурса + * которого нет, то мы не станем повторно спрашивать об этом удалённый репозиторий, а сразу ответим 404. + */ +public class Cache404 { + private static final long ONE_MIN = TimeUnit.MINUTES.toMillis(1); + private final Map map = new HashMap<>(); + private long lastCheck = System.currentTimeMillis(); + + private long actualTime; + + /** + * Создание кэша. + * + * @param actualTime актуальность кэша в минутах. Минимум 1 минута. + */ + public Cache404(int actualTime) { + if (actualTime < 1) { + actualTime = 1; + } + this.actualTime = TimeUnit.MINUTES.toMillis(actualTime); + } + + /** + * Проверяет, есть ли ресурс в кэше. + * + * @param path путь к ресурсу + * @return true, если ресурс есть в кэше + */ + public boolean contains(String path) { + checkOutTimes(); + Long exp = map.get(path); + if (exp == null) { + return false; + } else if ((System.currentTimeMillis() - exp) > actualTime) { + map.remove(path); + return false; + } + return true; + } + + /** + * Сохранить информацию об отсутствии ресурса. + * + * @param path путь к ресурсу + */ + public void put(String path) { + checkOutTimes(); + long exp = System.currentTimeMillis() + actualTime; + map.put(path, exp); + } + + /** + * Чистим кэш от устаревших данных. + * Не чаще одного раза в минуту. + */ + private void checkOutTimes() { + if ((System.currentTimeMillis() - lastCheck) <= ONE_MIN) { + return; + } + + long currTime = System.currentTimeMillis(); + map.entrySet().removeIf(e -> (currTime - e.getValue()) > actualTime); + lastCheck = currTime; + } +} diff --git a/src/main/java/ru/di9/mirror/utils/MavenClient.java b/src/main/java/ru/di9/mirror/utils/MavenClient.java new file mode 100644 index 0000000..17e4c16 --- /dev/null +++ b/src/main/java/ru/di9/mirror/utils/MavenClient.java @@ -0,0 +1,59 @@ +package ru.di9.mirror.utils; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; +import org.apache.hc.client5.http.classic.methods.HttpGet; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; + +/** + * Клиент для работы с Maven-like репозиториями. + */ +@Slf4j +public class MavenClient { + private final String baseUrl; + private final CloseableHttpClient httpClient; + + /** + * Создание клиента. + * + * @param url базовый URL репозитория + */ + public MavenClient(String url) { + this.baseUrl = url.endsWith("/") ? url.substring(0, url.length() - 1) : url; + this.httpClient = HttpClientBuilder.create().build(); + } + + /** + * Получить ресурс из репозитория. + * + * @param path путь к ресурсу + * @param outputStream стрим, в который загрузится ресурс + */ + public boolean getResource(String path, OutputStream outputStream) { + URI uri = URI.create(baseUrl + (path.startsWith("/") ? "" : "/") + path); + log.debug("GET '{}'", uri); + var httpGet = new HttpGet(uri); + + try { + return httpClient.execute(httpGet, resp -> { + if (resp.getCode() == 404) { + return false; + } else if (resp.getCode() != 200) { + log.error("HTTP GET '{}' RETURN CODE {}", uri, resp.getCode()); + return false; + } + + IOUtils.copy(resp.getEntity().getContent(), outputStream); + return true; + }); + } catch (IOException e) { + log.error("ERROR HTTP GET '{}'", uri, e); + return false; + } + } +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties new file mode 100644 index 0000000..93038dc --- /dev/null +++ b/src/main/resources/application.properties @@ -0,0 +1,8 @@ +server.address=0.0.0.0 +server.port=8080 + +app.maven.storage=storage + +app.maven.repository[0].id=central +app.maven.repository[0].url=https://repo1.maven.org/maven2 +app.maven.repository[0].cache-time=60 diff --git a/web-spring/build.gradle b/web-spring/build.gradle deleted file mode 100644 index 7562d08..0000000 --- a/web-spring/build.gradle +++ /dev/null @@ -1,29 +0,0 @@ -apply plugin: 'org.springframework.boot' -apply plugin: 'application' - -dependencies { - implementation(project(':core')) - - annotationProcessor(deps.springboot.config_processor) - implementation(deps.springboot.web) -} - -application { - mainClass = 'ru.di9.mirror.web.Application' -} - -tasks.named('compileJava') { - it.dependsOn('moveSpringConfigurationMetadata') -} - -tasks.register('moveSpringConfigurationMetadata').configure { - it.dependsOn('processResources') - doLast { - def metafile = file("${sourceSets.main.output.resourcesDir}/META-INF/additional-spring-configuration-metadata.json") - if (metafile.exists()) { - def metafileTo = file("${sourceSets.main.output.classesDirs.asPath}/META-INF/spring-configuration-metadata.json") - metafileTo.parentFile.mkdirs() - metafile.renameTo(metafileTo) - } - } -} diff --git a/web-spring/src/main/java/ru/di9/mirror/web/config/MavenMirrorsProperties.java b/web-spring/src/main/java/ru/di9/mirror/web/config/MavenMirrorsProperties.java deleted file mode 100644 index 8597396..0000000 --- a/web-spring/src/main/java/ru/di9/mirror/web/config/MavenMirrorsProperties.java +++ /dev/null @@ -1,24 +0,0 @@ -package ru.di9.mirror.web.config; - -import lombok.Getter; -import lombok.Setter; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -import java.util.ArrayList; -import java.util.List; - -@Component -@ConfigurationProperties(prefix = "maven-mirrors") -public class MavenMirrorsProperties { - - @Getter - @Setter - private String storage; - - @Getter - private final List list = new ArrayList<>(); - - public record MirrorData(String id, String url) { - } -} diff --git a/web-spring/src/main/java/ru/di9/mirror/web/config/WebConfig.java b/web-spring/src/main/java/ru/di9/mirror/web/config/WebConfig.java deleted file mode 100644 index 98cb33a..0000000 --- a/web-spring/src/main/java/ru/di9/mirror/web/config/WebConfig.java +++ /dev/null @@ -1,32 +0,0 @@ -package ru.di9.mirror.web.config; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import ru.di9.mirror.core.FileStorage; -import ru.di9.mirror.core.MavenClient; -import ru.di9.mirror.core.MirrorService; -import ru.di9.mirror.core.MirrorService.MirrorPair; - -import java.nio.file.Path; -import java.nio.file.Paths; - -@Slf4j -@Configuration -public class WebConfig { - - @Bean - @SuppressWarnings("java:S3864") - public MirrorService mirrorService(MavenMirrorsProperties mirrorsProperties) { - Path storagePath = Paths.get(mirrorsProperties.getStorage()); - - return new MirrorService(mirrorsProperties.getList() - .stream() - .peek(v -> log.info("read config mirror: {}|{}", v.id(), v.url())) - .map(mirrorData -> new MirrorPair( - new FileStorage(mirrorData.id(), storagePath), - new MavenClient(mirrorData.url()) - )) - .toList()); - } -} diff --git a/web-spring/src/main/java/ru/di9/mirror/web/controller/MavenController.java b/web-spring/src/main/java/ru/di9/mirror/web/controller/MavenController.java deleted file mode 100644 index ec998d3..0000000 --- a/web-spring/src/main/java/ru/di9/mirror/web/controller/MavenController.java +++ /dev/null @@ -1,36 +0,0 @@ -package ru.di9.mirror.web.controller; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.core.io.InputStreamResource; -import org.springframework.http.HttpHeaders; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.stereotype.Controller; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestMapping; -import ru.di9.mirror.core.MirrorService; - -@Slf4j -@Controller -@RequestMapping(path = "/maven") -public class MavenController { - - private final MirrorService mirrorService; - - public MavenController(MirrorService mirrorService) { - this.mirrorService = mirrorService; - } - - @GetMapping(path = "/{*path}") - public ResponseEntity artifactFileGet(@PathVariable("path") String httpPath) { - return mirrorService.getFile(httpPath) - .map(fileRecord -> ResponseEntity.ok() - .header(HttpHeaders.CONTENT_DISPOSITION, - "attachment; filename=\"" + fileRecord.name() + "\"") - .contentType(MediaType.APPLICATION_OCTET_STREAM) - .contentLength(fileRecord.size()) - .body(new InputStreamResource(fileRecord.inputStream()))) - .orElse(ResponseEntity.notFound().build()); - } -} diff --git a/web-spring/src/main/resources/META-INF/additional-spring-configuration-metadata.json b/web-spring/src/main/resources/META-INF/additional-spring-configuration-metadata.json deleted file mode 100644 index 65b3609..0000000 --- a/web-spring/src/main/resources/META-INF/additional-spring-configuration-metadata.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "properties": [ - { - "name": "maven-mirrors.storage", - "type": "java.lang.String" - }, - { - "name": "maven-mirrors.list", - "type": "java.util.List", - "sourceType": "java.util.List" - } - ] -} diff --git a/web-spring/src/main/resources/application.yml b/web-spring/src/main/resources/application.yml deleted file mode 100644 index 600b85c..0000000 --- a/web-spring/src/main/resources/application.yml +++ /dev/null @@ -1,11 +0,0 @@ -server: - address: 127.0.0.1 - port: 8080 - -debug: false - -maven-mirrors: - storage: './storage' - list: - - id: 'maven-central' - url: 'https://repo1.maven.org/maven2'