diff --git a/mirror-core/build.gradle b/mirror-core/build.gradle index 4b2369c..4f9127a 100644 --- a/mirror-core/build.gradle +++ b/mirror-core/build.gradle @@ -30,4 +30,5 @@ idea.module { dependencies { api("io.minio:minio:${minioVersion}") + api("org.mongodb:mongodb-driver-sync:${mongoDriver}") } diff --git a/mirror-core/src/main/java/ru/di9/mirror/core/domain/ArtifactRecord.java b/mirror-core/src/main/java/ru/di9/mirror/core/domain/ArtifactRecord.java index e51001d..f6185f4 100644 --- a/mirror-core/src/main/java/ru/di9/mirror/core/domain/ArtifactRecord.java +++ b/mirror-core/src/main/java/ru/di9/mirror/core/domain/ArtifactRecord.java @@ -1,7 +1,10 @@ package ru.di9.mirror.core.domain; +import lombok.extern.slf4j.Slf4j; + import java.util.StringJoiner; +@Slf4j public record ArtifactRecord(String group, String artifactId, String version, boolean isSnapshot) { private static final String SNAPSHOT = "SNAPSHOT"; @@ -9,6 +12,7 @@ public record ArtifactRecord(String group, String artifactId, String version, bo if (httpPath.startsWith("/")) { httpPath = httpPath.substring(1); } + log.info("httpPath = {}", httpPath); String[] partsPath = httpPath.split("/"); String version = partsPath[partsPath.length-2]; diff --git a/mirror-core/src/main/java/ru/di9/mirror/core/entity/ArtifactEntity.java b/mirror-core/src/main/java/ru/di9/mirror/core/entity/ArtifactEntity.java new file mode 100644 index 0000000..687c45e --- /dev/null +++ b/mirror-core/src/main/java/ru/di9/mirror/core/entity/ArtifactEntity.java @@ -0,0 +1,28 @@ +package ru.di9.mirror.core.entity; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.bson.types.ObjectId; +import ru.di9.mirror.core.domain.ArtifactRecord; + +@Data +@EqualsAndHashCode(exclude = {"id"}) +public class ArtifactEntity { + private ObjectId id; + + private String group; + private String artifactId; + private String version; + private boolean isSnapshot; + + public static ArtifactEntity fromRecord(ArtifactRecord artifactRecord) { + var entity = new ArtifactEntity(); + + entity.setGroup(artifactRecord.group()); + entity.setArtifactId(artifactRecord.artifactId()); + entity.setVersion(artifactRecord.version()); + entity.setSnapshot(artifactRecord.isSnapshot()); + + return entity; + } +} diff --git a/mirror-core/src/main/java/ru/di9/mirror/core/handler/MavenHandler.java b/mirror-core/src/main/java/ru/di9/mirror/core/handler/MavenHandler.java index 2d26c9d..93e2605 100644 --- a/mirror-core/src/main/java/ru/di9/mirror/core/handler/MavenHandler.java +++ b/mirror-core/src/main/java/ru/di9/mirror/core/handler/MavenHandler.java @@ -4,7 +4,9 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import ru.di9.mirror.core.domain.ArtifactRecord; import ru.di9.mirror.core.domain.FileInfo; +import ru.di9.mirror.core.entity.ArtifactEntity; import ru.di9.mirror.core.handler.response.GetFileResponse; +import ru.di9.mirror.core.repository.ArtifactRepository; import ru.di9.mirror.core.service.ExternalMavenService; import ru.di9.mirror.core.service.MinioService; @@ -18,14 +20,16 @@ public class MavenHandler { private final MinioService minioService; private final List externalMavenServices; + private final ArtifactRepository repository; public Optional getFile(String path) { var fileInfo = FileInfo.of(path.substring(path.lastIndexOf("/") + 1)); + ArtifactRecord artifactRecord = null; if (!fileInfo.fullName().equalsIgnoreCase("maven-metadata.xml") - || !fileInfo.ext().equalsIgnoreCase("sha1")) { + && !fileInfo.ext().equalsIgnoreCase("sha1")) { - ArtifactRecord artifactRecord = ArtifactRecord.fromHttpPath(path); + artifactRecord = ArtifactRecord.fromHttpPath(path); log.info(artifactRecord.toString()); } @@ -34,11 +38,11 @@ public class MavenHandler { return optionalInputStream .map(inputStream -> new GetFileResponse(fileInfo.name(), inputStream)); } else { - return findInMirrors(path, fileInfo.name()); + return findInMirrors(path, fileInfo.name(), artifactRecord); } } - private Optional findInMirrors(String path, String fileName) { + private Optional findInMirrors(String path, String fileName, ArtifactRecord artifactRecord) { Optional result; for (ExternalMavenService externalMavenService : externalMavenServices) { final String nameForStore = "/" + externalMavenService.getId() + "/" + path; @@ -50,6 +54,10 @@ public class MavenHandler { } else { result = externalMavenService.getFile(path); if (result.isPresent()) { + if (artifactRecord != null) { + repository.save(ArtifactEntity.fromRecord(artifactRecord)); + } + minioService.put(nameForStore, result.get()); return minioService.get(nameForStore) .map(inputStream -> new GetFileResponse(fileName, inputStream)); @@ -64,7 +72,7 @@ public class MavenHandler { var fileInfo = FileInfo.of(path.substring(path.lastIndexOf("/") + 1)); if (!fileInfo.fullName().equalsIgnoreCase("maven-metadata.xml") - || !fileInfo.ext().equalsIgnoreCase("sha1")) { + && !fileInfo.ext().equalsIgnoreCase("sha1")) { ArtifactRecord artifactRecord = ArtifactRecord.fromHttpPath(path); log.info(artifactRecord.toString()); diff --git a/mirror-core/src/main/java/ru/di9/mirror/core/repository/ArtifactRepository.java b/mirror-core/src/main/java/ru/di9/mirror/core/repository/ArtifactRepository.java new file mode 100644 index 0000000..ba3f61d --- /dev/null +++ b/mirror-core/src/main/java/ru/di9/mirror/core/repository/ArtifactRepository.java @@ -0,0 +1,32 @@ +package ru.di9.mirror.core.repository; + +import com.mongodb.client.MongoCollection; +import com.mongodb.client.result.InsertOneResult; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.bson.Document; +import org.bson.types.ObjectId; +import ru.di9.mirror.core.entity.ArtifactEntity; + +@Slf4j +@RequiredArgsConstructor +public class ArtifactRepository { + private final MongoCollection collection; + + public void save(ArtifactEntity artifactEntity) { + if (artifactEntity.getId() == null) { + insert(artifactEntity); + } + } + + private void insert(ArtifactEntity artifactEntity) { + InsertOneResult insertOneResult = collection.insertOne(new Document() + .append("_id", new ObjectId()) + .append("group", artifactEntity.getGroup()) + .append("artifactId", artifactEntity.getArtifactId()) + .append("version", artifactEntity.getVersion()) + .append("is_snapshot", artifactEntity.isSnapshot())); + + log.info("InsertedId: {}", insertOneResult.getInsertedId()); + } +} diff --git a/mirror-web/src/main/java/ru/di9/mirror/web/config/WebConfig.java b/mirror-web/src/main/java/ru/di9/mirror/web/config/WebConfig.java index 1d504e1..d58d180 100644 --- a/mirror-web/src/main/java/ru/di9/mirror/web/config/WebConfig.java +++ b/mirror-web/src/main/java/ru/di9/mirror/web/config/WebConfig.java @@ -1,12 +1,20 @@ package ru.di9.mirror.web.config; +import com.mongodb.MongoClientSettings; +import com.mongodb.ServerAddress; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; import io.minio.MinioClient; import okhttp3.OkHttpClient; +import org.bson.Document; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import ru.di9.mirror.core.handler.IndexOfHandler; import ru.di9.mirror.core.handler.MavenHandler; +import ru.di9.mirror.core.repository.ArtifactRepository; import ru.di9.mirror.core.service.ExternalMavenService; import ru.di9.mirror.core.service.MinioService; @@ -17,8 +25,9 @@ public class WebConfig { @Bean public MavenHandler mavenHandler(MinioService minioService, - List externalMavenServices) { - return new MavenHandler(minioService, externalMavenServices); + List externalMavenServices, + ArtifactRepository artifactRepository) { + return new MavenHandler(minioService, externalMavenServices, artifactRepository); } @Bean @@ -54,4 +63,26 @@ public class WebConfig { return new OkHttpClient(); } + @Bean + public MongoClient mongoClient(@Value("${mongodb.host}") String host, @Value("${mongodb.port}") int port) { + return MongoClients.create(MongoClientSettings.builder() + .applyToClusterSettings(builder -> builder.hosts(List.of(new ServerAddress(host, port)))) + .build()); + } + + @Bean + public MongoDatabase mongoDatabase(MongoClient mongoClient, + @Value("${mongodb.database}") String database) { + return mongoClient.getDatabase(database); + } + + @Bean + public MongoCollection artifactCollection(MongoDatabase database) { + return database.getCollection("artifacts"); + } + + @Bean + public ArtifactRepository artifactRepository(MongoCollection artifactCollection) { + return new ArtifactRepository(artifactCollection); + } } diff --git a/mirror-web/src/main/resources/META-INF/additional-spring-configuration-metadata.json b/mirror-web/src/main/resources/META-INF/additional-spring-configuration-metadata.json index 5f8640c..eee77af 100644 --- a/mirror-web/src/main/resources/META-INF/additional-spring-configuration-metadata.json +++ b/mirror-web/src/main/resources/META-INF/additional-spring-configuration-metadata.json @@ -24,5 +24,21 @@ "name": "maven-mirrors.list", "type": "java.util.List", "sourceType": "java.util.List" - } -] } + }, + { + "name": "mongodb.host", + "type": "java.lang.String", + "description": "MongoDB host." + }, + { + "name": "mongodb.port", + "type": "java.lang.Integer", + "description": "MongoDB port." + }, + { + "name": "mongodb.database", + "type": "java.lang.String", + "description": "MongoDB database name." + } + ] +} diff --git a/mirror-web/src/main/resources/application.yml b/mirror-web/src/main/resources/application.yml index 0b74f11..1d28ab7 100644 --- a/mirror-web/src/main/resources/application.yml +++ b/mirror-web/src/main/resources/application.yml @@ -15,6 +15,11 @@ minio: secretKey: 'mirror123' bucket: 'mirror' +mongodb: + host: 'dev.di9.ru' + port: 27017 + database: 'mirror' + maven-mirrors: list: - id: 'maven_central' diff --git a/versions.gradle b/versions.gradle index d5bdfcb..7ec9546 100644 --- a/versions.gradle +++ b/versions.gradle @@ -3,6 +3,7 @@ ext { slf4jVersion = '1.7.36' minioVersion = '8.3.9' springBootVerson = '2.6.6' + mongoDriver = '4.6.0' // for tests only junitJupiterVersion = '5.5.2'