WIP

mirror-core/build.gradle

@@ -3,4 +3,7 @@ apply plugin: 'java-library'
 dependencies {
     api("com.squareup.okhttp3:okhttp:${okhttpVersion}")
     implementation("commons-io:commons-io:${commonsIoVersion}")
+    implementation("commons-codec:commons-codec:${commonsCodecVersion}")
+
+    testImplementation("org.slf4j:slf4j-simple:${slf4jVersion}")
 }

mirror-core/src/main/java/ru/di9/mirror/core/dao/ArtifactDao.java

@@ -1,8 +1,15 @@
 package ru.di9.mirror.core.dao;
 
+import ru.di9.mirror.core.domain.ArtifactDefinition;
 import ru.di9.mirror.core.entity.ArtifactEntity;
 
+import java.util.Optional;
+
 public interface ArtifactDao {
 
+    default Optional<ArtifactEntity> findByDefinition(ArtifactDefinition definition) {
+        return Optional.empty();
+    }
+
     ArtifactEntity save(ArtifactEntity artifactEntity);
 }

mirror-core/src/main/java/ru/di9/mirror/core/domain/ArtifactDefinition.java

@@ -0,0 +1,31 @@
+package ru.di9.mirror.core.domain;
+
+import java.util.StringJoiner;
+
+public record ArtifactDefinition(String group, String name, String version) {
+    private static final String SNAPSHOT = "SNAPSHOT";
+
+    public static ArtifactDefinition fromPath(String path) {
+        if (path.startsWith("/")) {
+            path = path.substring(1);
+        }
+
+        String[] partsPath = path.split("/");
+        String version = partsPath[partsPath.length - 2];
+        String fileName = partsPath[partsPath.length - 1];
+
+        String artifactId;
+        if (version.contains(SNAPSHOT)) {
+            artifactId = fileName.substring(0, fileName.indexOf(version.substring(0, version.lastIndexOf(SNAPSHOT))) - 1);
+        } else {
+            artifactId = fileName.substring(0, (fileName.lastIndexOf('.') - version.length() - 1));
+        }
+
+        StringJoiner group = new StringJoiner(".");
+        for (int i = 0; i < partsPath.length - 3; i++) {
+            group.add(partsPath[i]);
+        }
+
+        return new ArtifactDefinition(group.toString(), artifactId, version);
+    }
+}

mirror-core/src/main/java/ru/di9/mirror/core/entity/ArtifactEntity.java

@@ -1,49 +1,16 @@
 package ru.di9.mirror.core.entity;
 
-import lombok.AllArgsConstructor;
 import lombok.Data;
-import lombok.NoArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
 
-import java.util.StringJoiner;
-
-@Slf4j
-@AllArgsConstructor
-@NoArgsConstructor
 @Data
 public class ArtifactEntity {
-    private static final String SNAPSHOT = "SNAPSHOT";
 
     private String group;
     private String artifactId;
     private String version;
-    private boolean isSnapshot;
 
-    public static ArtifactEntity fromHttpPath(String httpPath) {
+    private String sha1Jar;
-        if (httpPath.startsWith("/")) {
+    private String sha1Pom;
-            httpPath = httpPath.substring(1);
-        }
-        log.info("httpPath = {}", httpPath);
 
-        String[] partsPath = httpPath.split("/");
+    private String store;
-        String version = partsPath[partsPath.length-2];
-        boolean isSnapshot = version.contains(SNAPSHOT);
-
-        String fileName = partsPath[partsPath.length-1];
-
-        String artifactId;
-        if (isSnapshot) {
-            String partVersion = version.substring(0, version.lastIndexOf(SNAPSHOT));
-            artifactId = fileName.substring(0, fileName.indexOf(partVersion) - 1);
-        } else {
-            artifactId = fileName.substring(0, (fileName.lastIndexOf('.') - version.length() - 1));
-        }
-
-        StringJoiner sj = new StringJoiner(".");
-        for (int i = 0; i < partsPath.length - 3; i++) {
-            sj.add(partsPath[i]);
-        }
-
-        return new ArtifactEntity(sj.toString(), artifactId, version, isSnapshot);
-    }
 }

mirror-core/src/main/java/ru/di9/mirror/core/handler/IndexOfHandler.java

@@ -4,6 +4,7 @@ import lombok.RequiredArgsConstructor;
 import ru.di9.mirror.core.domain.FileItem;
 import ru.di9.mirror.core.storage.FileStorage;
 
+import java.util.Collections;
 import java.util.List;
 
 @RequiredArgsConstructor
@@ -12,7 +13,7 @@ public class IndexOfHandler {
     private final FileStorage fileStorage;
 
     public List<FileItem> walker(String path) {
-        return fileStorage.list("/local/" + path);
+        return Collections.emptyList();//fileStorage.list("/local/" + path);
     }
 
 }

mirror-core/src/main/java/ru/di9/mirror/core/handler/MavenHandler.java

@@ -2,16 +2,26 @@ package ru.di9.mirror.core.handler;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.codec.digest.MessageDigestAlgorithms;
 import org.apache.commons.io.IOUtils;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+import ru.di9.mirror.core.domain.ArtifactDefinition;
 import ru.di9.mirror.core.domain.FileInfo;
 import ru.di9.mirror.core.entity.ArtifactEntity;
 import ru.di9.mirror.core.handler.response.GetFileResponse;
 import ru.di9.mirror.core.dao.ArtifactDao;
 import ru.di9.mirror.core.storage.FileStorage;
 import ru.di9.mirror.core.service.ExternalMavenService;
+import ru.di9.mirror.core.util.FileInfoWrapper;
+import ru.di9.mirror.core.util.Utils;
 
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
 import java.util.List;
 import java.util.Optional;
 
@@ -19,75 +29,212 @@ import java.util.Optional;
 @RequiredArgsConstructor
 public class MavenHandler {
 
+    private final ArtifactDao artifactDao;
     private final FileStorage fileStorage;
     private final List<ExternalMavenService> externalMavenServices;
-    private final ArtifactDao repository;
 
-    public Optional<GetFileResponse> getFile(String path) {
+    public Optional<GetFileResponse> getArtifactFile(String path) {
-        var fileInfo = FileInfo.of(path.substring(path.lastIndexOf("/") + 1));
+        var fileInfo = new FileInfoWrapper(path.substring(path.lastIndexOf("/") + 1));
-
+        String ext = fileInfo.ext();
-        ArtifactEntity artifactEntity = null;
+        if (!ext.equalsIgnoreCase("jar")
-        if (!fileInfo.fullName().equalsIgnoreCase("maven-metadata.xml")
+            && !ext.equalsIgnoreCase("pom")
-            && !fileInfo.ext().equalsIgnoreCase("sha1")) {
+            && !ext.equalsIgnoreCase("sha1")) {
-
+            return Optional.empty();
-            artifactEntity = ArtifactEntity.fromHttpPath(path);
-            log.info(artifactEntity.toString());
         }
 
-        Optional<InputStream> optionalInputStream = fileStorage.findByName("/local/" + path);
+        var artifactDefinition = ArtifactDefinition.fromPath(path);
-        if (optionalInputStream.isPresent()) {
+        Optional<ArtifactEntity> entity = artifactDao.findByDefinition(artifactDefinition);
-            return optionalInputStream
+        if (entity.isPresent()) {
-                .map(inputStream -> new GetFileResponse(fileInfo.name(), inputStream));
+            return processExistsArtifact(ext, path, fileInfo, entity.get());
         } else {
-            return findInMirrors(path, fileInfo, artifactEntity);
+            if (ext.equalsIgnoreCase("jar") || ext.equalsIgnoreCase("pom")) {
-        }
+                return processJarPom2(path, fileInfo, ext, artifactDefinition);
-    }
+            } else if (ext.equalsIgnoreCase("sha1")) {
-
+                return Optional.empty();
-    private Optional<GetFileResponse> findInMirrors(String path, FileInfo fileInfo, ArtifactEntity artifactEntity) {
-        Optional<InputStream> result;
-        for (ExternalMavenService externalMavenService : externalMavenServices) {
-            final String nameForStore = "/" + externalMavenService.getId() + "/" + path;
-
-            result = fileStorage.findByName(nameForStore);
-            if (result.isPresent()) {
-                return result
-                    .map(inputStream -> new GetFileResponse(fileInfo.name(), inputStream));
-            } else {
-                result = externalMavenService.getFile(path);
-                if (result.isPresent()) {
-                    if (artifactEntity != null) {
-                        repository.save(artifactEntity);
-                    }
-
-                    if (fileInfo.ext().equalsIgnoreCase("sha1")) {
-                        try {
-                            String sha1str = IOUtils.toString(result.get(), StandardCharsets.UTF_8);
-                            fileStorage.setMetadata(nameForStore.substring(0, nameForStore.lastIndexOf(".")), "sha1", sha1str);
-                        } catch (Exception e) {
-                            throw new RuntimeException(e);
-                        }
-                    } else {
-                        fileStorage.save(nameForStore, result.get());
-                    }
-                    return fileStorage.findByName(nameForStore)
-                        .map(inputStream -> new GetFileResponse(fileInfo.name(), inputStream));
-                }
             }
         }
 
         return Optional.empty();
     }
 
-    public void putFile(String path, InputStream inputStream) {
+    private Optional<GetFileResponse> processJarPom2(String path, FileInfoWrapper fileInfo, String ext, ArtifactDefinition artifactDefinition) {
-        var fileInfo = FileInfo.of(path.substring(path.lastIndexOf("/") + 1));
+        for (ExternalMavenService externalMavenService : externalMavenServices) {
+            Optional<InputStream> inputStream = externalMavenService.getFile(path);
+            if (inputStream.isEmpty()) {
+                continue;
+            }
 
-        if (!fileInfo.fullName().equalsIgnoreCase("maven-metadata.xml")
+            byte[] bytes;
-            && !fileInfo.ext().equalsIgnoreCase("sha1")) {
+            try {
+                bytes = IOUtils.toByteArray(inputStream.get());
+                inputStream.get().close();
+            } catch (IOException e) {
+                log.error(e.getMessage());
+                throw new RuntimeException(e);
+            }
 
-            ArtifactEntity artifactEntity = ArtifactEntity.fromHttpPath(path);
+            fileStorage.put(path, new ByteArrayInputStream(bytes));
-            log.info(artifactEntity.toString());
+
+            var artifactEntity = new ArtifactEntity();
+            artifactEntity.setGroup(artifactDefinition.group());
+            artifactEntity.setArtifactId(artifactDefinition.name());
+            artifactEntity.setVersion(artifactDefinition.version());
+            artifactEntity.setStore(externalMavenService.getId());
+            setSha1(ext, artifactEntity, new DigestUtils(MessageDigestAlgorithms.SHA_1).digestAsHex(bytes));
+            artifactDao.save(artifactEntity);
+
+            return Optional.of(new GetFileResponse(fileInfo.fullName(), new ByteArrayInputStream(bytes)));
         }
 
-        fileStorage.save("/local/" + path, inputStream);
+        return Optional.empty();
+    }
+
+    private Optional<GetFileResponse> processExistsArtifact(String ext, String path,
+                                                            FileInfoWrapper fileInfo, ArtifactEntity entity) {
+        if (ext.equalsIgnoreCase("jar") || ext.equalsIgnoreCase("pom")) {
+            return processJarPom(path, fileInfo.fullName(), ext, entity);
+        } else if (ext.equalsIgnoreCase("sha1")) {
+            return processSha1(fileInfo, entity);
+        }
+
+        return Optional.empty();
+    }
+
+    private Optional<GetFileResponse> processSha1(FileInfoWrapper fileInfo, ArtifactEntity entity) {
+        var fileInfo2 = new FileInfoWrapper(fileInfo.name());
+
+        if (fileInfo2.ext().equalsIgnoreCase("jar")) {
+            return Optional.of(new GetFileResponse(fileInfo.fullName(), IOUtils.toInputStream(entity.getSha1Jar(), StandardCharsets.UTF_8)));
+        } else if (fileInfo2.ext().equalsIgnoreCase("pom")) {
+            return Optional.of(new GetFileResponse(fileInfo.fullName(), IOUtils.toInputStream(entity.getSha1Pom(), StandardCharsets.UTF_8)));
+        } else {
+            return Optional.empty();
+        }
+    }
+
+    private Optional<GetFileResponse> processJarPom(String path, String fullFileName, String ext, ArtifactEntity entity) {
+        if (Utils.isNotEmptyString(getSha1(ext, entity))) {
+            return Optional.of(new GetFileResponse(fullFileName, fileStorage.get(path)));
+        }
+
+        Optional<ExternalMavenService> externalMavenService = externalMavenServices.stream()
+            .filter(ems -> ems.getId().equalsIgnoreCase(entity.getStore()))
+            .findFirst();
+
+        if (externalMavenService.isPresent()) {
+            return getFileFromExternalMavenService(path, fullFileName, ext, entity, externalMavenService.get());
+        } else {
+            log.warn("Unknown store: {}", entity.getStore());
+            return Optional.empty();
+        }
+    }
+
+    private Optional<GetFileResponse> getFileFromExternalMavenService(String path, String fullFileName, String ext, ArtifactEntity entity, ExternalMavenService externalMavenService) {
+        return externalMavenService.getFile(path)
+            .map(inputStream -> {
+                byte[] bytes = null;
+                try {
+                    bytes = IOUtils.toByteArray(inputStream);
+                    inputStream.close();
+                } catch (IOException e) {
+                    log.error(e.getMessage(), e);
+                }
+                return bytes;
+            })
+            .map(bytes -> {
+                fileStorage.put(path, new ByteArrayInputStream(bytes));
+                setSha1(ext, entity, new DigestUtils(MessageDigestAlgorithms.SHA_1).digestAsHex(bytes));
+                artifactDao.save(entity);
+
+                return new GetFileResponse(fullFileName, new ByteArrayInputStream(bytes));
+            });
+    }
+
+
+    private String getSha1(String type, ArtifactEntity entity) {
+        if (type.equalsIgnoreCase("jar")) {
+            return entity.getSha1Jar();
+        } else if (type.equalsIgnoreCase("pom")) {
+            return entity.getSha1Pom();
+        } else {
+            return null;
+        }
+    }
+
+    private void setSha1(String type, ArtifactEntity entity, String value) {
+        if (type.equalsIgnoreCase("jar")) {
+            entity.setSha1Jar(value);
+        } else if (type.equalsIgnoreCase("pom")) {
+            entity.setSha1Pom(value);
+        }
+    }
+
+    public Optional<GetFileResponse> getFile(String path) {
+//        var fileInfo = FileInfo.of(path.substring(path.lastIndexOf("/") + 1));
+//
+//        ArtifactEntity artifactEntity = null;
+//        if (!fileInfo.fullName().equalsIgnoreCase("maven-metadata.xml")
+//            && !fileInfo.ext().equalsIgnoreCase("sha1")) {
+//
+//            artifactEntity = ArtifactEntity.fromHttpPath(path);
+//            log.info(artifactEntity.toString());
+//        }
+//
+//        Optional<InputStream> optionalInputStream = fileStorage.findByName("/local/" + path);
+//        if (optionalInputStream.isPresent()) {
+//            return optionalInputStream
+//                .map(inputStream -> new GetFileResponse(fileInfo.name(), inputStream));
+//        } else {
+//            return findInMirrors(path, fileInfo, artifactEntity);
+//        }
+        return null;
+    }
+
+    private Optional<GetFileResponse> findInMirrors(String path, FileInfo fileInfo, ArtifactEntity artifactEntity) {
+//        Optional<InputStream> result;
+//        for (ExternalMavenService externalMavenService : externalMavenServices) {
+//            final String nameForStore = "/" + externalMavenService.getId() + "/" + path;
+//
+//            result = fileStorage.findByName(nameForStore);
+//            if (result.isPresent()) {
+//                return result
+//                    .map(inputStream -> new GetFileResponse(fileInfo.name(), inputStream));
+//            } else {
+//                result = externalMavenService.getFile(path);
+//                if (result.isPresent()) {
+//                    if (artifactEntity != null) {
+//                        artifactDao.save(artifactEntity);
+//                    }
+//
+//                    if (fileInfo.ext().equalsIgnoreCase("sha1")) {
+//                        try {
+//                            String sha1str = IOUtils.toString(result.get(), StandardCharsets.UTF_8);
+//                            fileStorage.setMetadata(nameForStore.substring(0, nameForStore.lastIndexOf(".")), "sha1", sha1str);
+//                        } catch (Exception e) {
+//                            throw new RuntimeException(e);
+//                        }
+//                    } else {
+//                        fileStorage.save(nameForStore, result.get());
+//                    }
+//                    return fileStorage.findByName(nameForStore)
+//                        .map(inputStream -> new GetFileResponse(fileInfo.name(), inputStream));
+//                }
+//            }
+//        }
+
+        return Optional.empty();
+    }
+
+    public void putFile(String path, InputStream inputStream) {
+//        var fileInfo = FileInfo.of(path.substring(path.lastIndexOf("/") + 1));
+//
+//        if (!fileInfo.fullName().equalsIgnoreCase("maven-metadata.xml")
+//            && !fileInfo.ext().equalsIgnoreCase("sha1")) {
+//
+//            ArtifactEntity artifactEntity = ArtifactEntity.fromHttpPath(path);
+//            log.info(artifactEntity.toString());
+//        }
+//
+//        fileStorage.save("/local/" + path, inputStream);
     }
 }

mirror-core/src/main/java/ru/di9/mirror/core/storage/FileStorage.java

@@ -1,18 +1,10 @@
 package ru.di9.mirror.core.storage;
 
-import ru.di9.mirror.core.domain.FileItem;
-
 import java.io.InputStream;
-import java.util.List;
-import java.util.Optional;
 
 public interface FileStorage {
 
-    Optional<InputStream> findByName(String name);
+    InputStream get(String name);
 
-    void save(String name, InputStream inputStream);
+    void put(String name, InputStream inputStream);
-
-    void setMetadata(String name, String key, String value);
-
-    List<FileItem> list(String prefix);
 }

mirror-core/src/main/java/ru/di9/mirror/core/util/FileInfoWrapper.java

@@ -0,0 +1,12 @@
+package ru.di9.mirror.core.util;
+
+public record FileInfoWrapper(String fullName) {
+
+    public String name() {
+        return fullName.substring(0, fullName.lastIndexOf('.'));
+    }
+
+    public String ext() {
+        return fullName.substring(fullName.lastIndexOf('.') + 1);
+    }
+}

mirror-core/src/main/java/ru/di9/mirror/core/util/Utils.java

@@ -1,4 +1,4 @@
-package ru.di9.mirror.core;
+package ru.di9.mirror.core.util;
 
 import lombok.experimental.UtilityClass;
 

mirror-core/src/test/java/ru/di9/mirror/core/entity/ArtifactEntityTest.java

@@ -1,29 +1,29 @@
-package ru.di9.mirror.core.entity;
+//package ru.di9.mirror.core.entity;
-
+//
-import org.junit.jupiter.api.Assertions;
+//import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.params.ParameterizedTest;
+//import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.Arguments;
+//import org.junit.jupiter.params.provider.Arguments;
-import org.junit.jupiter.params.provider.MethodSource;
+//import org.junit.jupiter.params.provider.MethodSource;
-
+//
-import java.util.stream.Stream;
+//import java.util.stream.Stream;
-
+//
-class ArtifactEntityTest {
+//class ArtifactEntityTest {
-
+//
-    @ParameterizedTest(name = "[{index}] {0}")
+//    @ParameterizedTest(name = "[{index}] {0}")
-    @MethodSource("fromHttpPathSource")
+//    @MethodSource("fromHttpPathSource")
-    void fromHttpPath(String httpPath, ArtifactEntity expected) {
+//    void fromHttpPath(String httpPath, ArtifactEntity expected) {
-        ArtifactEntity actual = ArtifactEntity.fromHttpPath(httpPath);
+//        ArtifactEntity actual = ArtifactEntity.fromHttpPath(httpPath);
-        Assertions.assertEquals(expected, actual);
+//        Assertions.assertEquals(expected, actual);
-    }
+//    }
-
+//
-    static Stream<Arguments> fromHttpPathSource() {
+//    static Stream<Arguments> fromHttpPathSource() {
-        return Stream.of(
+//        return Stream.of(
-            Arguments.of("com/google/guava/guava/21.0/guava-21.0.jar",
+//            Arguments.of("com/google/guava/guava/21.0/guava-21.0.jar",
-                new ArtifactEntity("com.google.guava", "guava", "21.0", false)),
+//                new ArtifactEntity("com.google.guava", "guava", "21.0", false)),
-            Arguments.of("org/bukkit/bukkit/1.12.2-R0.1-SNAPSHOT/bukkit-1.12.2-R0.1-20180712.012114-155.jar",
+//            Arguments.of("org/bukkit/bukkit/1.12.2-R0.1-SNAPSHOT/bukkit-1.12.2-R0.1-20180712.012114-155.jar",
-                new ArtifactEntity("org.bukkit", "bukkit", "1.12.2-R0.1-SNAPSHOT", true)),
+//                new ArtifactEntity("org.bukkit", "bukkit", "1.12.2-R0.1-SNAPSHOT", true)),
-            Arguments.of("/org/projectlombok/lombok/1.18.22/lombok-1.18.22.jar",
+//            Arguments.of("/org/projectlombok/lombok/1.18.22/lombok-1.18.22.jar",
-                new ArtifactEntity("org.projectlombok", "lombok", "1.18.22", false))
+//                new ArtifactEntity("org.projectlombok", "lombok", "1.18.22", false))
-        );
+//        );
-    }
+//    }
-}
+//}

mirror-core/src/test/java/ru/di9/mirror/core/handler/MavenHandlerTest.java

@@ -0,0 +1,24 @@
+package ru.di9.mirror.core.handler;
+
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import ru.di9.mirror.core.handler.response.GetFileResponse;
+
+import java.util.Optional;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+class MavenHandlerTest {
+
+    @Test
+    @DisplayName("В БД есть артефакт")
+    void existsArtifact() {
+        MavenHandler mavenHandler = new MavenHandler(null, null, null);
+        Optional<GetFileResponse> artifactFile = mavenHandler.getArtifactFile("ghast/ghast-tools/1.13/ghast-tools-1.13.jar");
+
+        assertNotNull(artifactFile);
+        assertTrue(artifactFile.isPresent());
+        assertEquals("ghast-tools-1.13.jar", artifactFile.get().name());
+        assertNotNull(artifactFile.get().inputStream());
+    }
+}

mirror-web/src/main/java/ru/di9/mirror/web/config/WebConfig.java

@@ -18,7 +18,7 @@ public class WebConfig {
     public MavenHandler mavenHandler(FileStorage fileStorage,
                                      List<ExternalMavenService> externalMavenServices,
                                      ArtifactDao artifactDao) {
-        return new MavenHandler(fileStorage, externalMavenServices, artifactDao);
+        return new MavenHandler(artifactDao, fileStorage, externalMavenServices);
     }
 
     @Bean

versions.gradle

@@ -6,6 +6,7 @@ ext {
     mongoDriver = '4.6.0'
     okhttpVersion = '4.8.1'
     commonsIoVersion = '2.11.0'
+    commonsCodecVersion = '1.15'
 
     // for tests only
     junitJupiterVersion = '5.5.2'