Voomra/Project-Mirror
0
Fork 0
Code Issues 2 Releases Activity

добавлена MongoDB для хранения данных об артефактах

Browse Source 
see #3
This commit is contained in:
DmitriyMX
2022-05-09 16:17:46 +03:00
parent 420c62f885
commit 1b48647c39
9 changed files with 135 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
mirror-core/build.gradle
View File

@@ -30,4 +30,5 @@ idea.module {
dependencies {
api("io.minio:minio:${minioVersion}")
api("org.mongodb:mongodb-driver-sync:${mongoDriver}")
}

4
mirror-core/src/main/java/ru/di9/mirror/core/domain/ArtifactRecord.java
View File

@@ -1,7 +1,10 @@
package ru.di9.mirror.core.domain;
import lombok.extern.slf4j.Slf4j;
import java.util.StringJoiner;
@Slf4j
public record ArtifactRecord(String group, String artifactId, String version, boolean isSnapshot) {
private static final String SNAPSHOT = "SNAPSHOT";
@@ -9,6 +12,7 @@ public record ArtifactRecord(String group, String artifactId, String version, bo
if (httpPath.startsWith("/")) {
httpPath = httpPath.substring(1);
}
log.info("httpPath = {}", httpPath);
String[] partsPath = httpPath.split("/");
String version = partsPath[partsPath.length-2];

28
mirror-core/src/main/java/ru/di9/mirror/core/entity/ArtifactEntity.java Normal file
View File

@@ -0,0 +1,28 @@
package ru.di9.mirror.core.entity;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.bson.types.ObjectId;
import ru.di9.mirror.core.domain.ArtifactRecord;
@Data
@EqualsAndHashCode(exclude = {"id"})
public class ArtifactEntity {
private ObjectId id;
private String group;
private String artifactId;
private String version;
private boolean isSnapshot;
public static ArtifactEntity fromRecord(ArtifactRecord artifactRecord) {
var entity = new ArtifactEntity();
entity.setGroup(artifactRecord.group());
entity.setArtifactId(artifactRecord.artifactId());
entity.setVersion(artifactRecord.version());
entity.setSnapshot(artifactRecord.isSnapshot());
return entity;
}
}

18
mirror-core/src/main/java/ru/di9/mirror/core/handler/MavenHandler.java
View File

@@ -4,7 +4,9 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import ru.di9.mirror.core.domain.ArtifactRecord;
import ru.di9.mirror.core.domain.FileInfo;
import ru.di9.mirror.core.entity.ArtifactEntity;
import ru.di9.mirror.core.handler.response.GetFileResponse;
import ru.di9.mirror.core.repository.ArtifactRepository;
import ru.di9.mirror.core.service.ExternalMavenService;
import ru.di9.mirror.core.service.MinioService;
@@ -18,14 +20,16 @@ public class MavenHandler {
private final MinioService minioService;
private final List<ExternalMavenService> externalMavenServices;
private final ArtifactRepository repository;
public Optional<GetFileResponse> getFile(String path) {
var fileInfo = FileInfo.of(path.substring(path.lastIndexOf("/") + 1));
ArtifactRecord artifactRecord = null;
if (!fileInfo.fullName().equalsIgnoreCase("maven-metadata.xml")
|| !fileInfo.ext().equalsIgnoreCase("sha1")) {
&& !fileInfo.ext().equalsIgnoreCase("sha1")) {
ArtifactRecord artifactRecord = ArtifactRecord.fromHttpPath(path);
artifactRecord = ArtifactRecord.fromHttpPath(path);
log.info(artifactRecord.toString());
}
@@ -34,11 +38,11 @@ public class MavenHandler {
return optionalInputStream
.map(inputStream -> new GetFileResponse(fileInfo.name(), inputStream));
} else {
return findInMirrors(path, fileInfo.name());
return findInMirrors(path, fileInfo.name(), artifactRecord);
}
}
private Optional<GetFileResponse> findInMirrors(String path, String fileName) {
private Optional<GetFileResponse> findInMirrors(String path, String fileName, ArtifactRecord artifactRecord) {
Optional<InputStream> result;
for (ExternalMavenService externalMavenService : externalMavenServices) {
final String nameForStore = "/" + externalMavenService.getId() + "/" + path;
@@ -50,6 +54,10 @@ public class MavenHandler {
} else {
result = externalMavenService.getFile(path);
if (result.isPresent()) {
if (artifactRecord != null) {
repository.save(ArtifactEntity.fromRecord(artifactRecord));
}
minioService.put(nameForStore, result.get());
return minioService.get(nameForStore)
.map(inputStream -> new GetFileResponse(fileName, inputStream));
@@ -64,7 +72,7 @@ public class MavenHandler {
var fileInfo = FileInfo.of(path.substring(path.lastIndexOf("/") + 1));
if (!fileInfo.fullName().equalsIgnoreCase("maven-metadata.xml")
|| !fileInfo.ext().equalsIgnoreCase("sha1")) {
&& !fileInfo.ext().equalsIgnoreCase("sha1")) {
ArtifactRecord artifactRecord = ArtifactRecord.fromHttpPath(path);
log.info(artifactRecord.toString());

32
mirror-core/src/main/java/ru/di9/mirror/core/repository/ArtifactRepository.java Normal file
View File

@@ -0,0 +1,32 @@
package ru.di9.mirror.core.repository;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.result.InsertOneResult;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.bson.Document;
import org.bson.types.ObjectId;
import ru.di9.mirror.core.entity.ArtifactEntity;
@Slf4j
@RequiredArgsConstructor
public class ArtifactRepository {
private final MongoCollection<Document> collection;
public void save(ArtifactEntity artifactEntity) {
if (artifactEntity.getId() == null) {
insert(artifactEntity);
}
}
private void insert(ArtifactEntity artifactEntity) {
InsertOneResult insertOneResult = collection.insertOne(new Document()
.append("_id", new ObjectId())
.append("group", artifactEntity.getGroup())
.append("artifactId", artifactEntity.getArtifactId())
.append("version", artifactEntity.getVersion())
.append("is_snapshot", artifactEntity.isSnapshot()));
log.info("InsertedId: {}", insertOneResult.getInsertedId());
}
}

35
mirror-web/src/main/java/ru/di9/mirror/web/config/WebConfig.java
View File

@@ -1,12 +1,20 @@
package ru.di9.mirror.web.config;
import com.mongodb.MongoClientSettings;
import com.mongodb.ServerAddress;
import com.mongodb.client.MongoClient;
import com.mongodb.client.MongoClients;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import io.minio.MinioClient;
import okhttp3.OkHttpClient;
import org.bson.Document;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import ru.di9.mirror.core.handler.IndexOfHandler;
import ru.di9.mirror.core.handler.MavenHandler;
import ru.di9.mirror.core.repository.ArtifactRepository;
import ru.di9.mirror.core.service.ExternalMavenService;
import ru.di9.mirror.core.service.MinioService;
@@ -17,8 +25,9 @@ public class WebConfig {
@Bean
public MavenHandler mavenHandler(MinioService minioService,
List<ExternalMavenService> externalMavenServices) {
return new MavenHandler(minioService, externalMavenServices);
List<ExternalMavenService> externalMavenServices,
ArtifactRepository artifactRepository) {
return new MavenHandler(minioService, externalMavenServices, artifactRepository);
}
@Bean
@@ -54,4 +63,26 @@ public class WebConfig {
return new OkHttpClient();
}
@Bean
public MongoClient mongoClient(@Value("${mongodb.host}") String host, @Value("${mongodb.port}") int port) {
return MongoClients.create(MongoClientSettings.builder()
.applyToClusterSettings(builder -> builder.hosts(List.of(new ServerAddress(host, port))))
.build());
}
@Bean
public MongoDatabase mongoDatabase(MongoClient mongoClient,
@Value("${mongodb.database}") String database) {
return mongoClient.getDatabase(database);
}
@Bean
public MongoCollection<Document> artifactCollection(MongoDatabase database) {
return database.getCollection("artifacts");
}
@Bean
public ArtifactRepository artifactRepository(MongoCollection<Document> artifactCollection) {
return new ArtifactRepository(artifactCollection);
}
}

20
mirror-web/src/main/resources/META-INF/additional-spring-configuration-metadata.json
View File

@@ -24,5 +24,21 @@
"name": "maven-mirrors.list",
"type": "java.util.List<ru.di9.mirror.web.config.MavenMirrorsProperties.MirrorData>",
"sourceType": "java.util.List<ru.di9.mirror.web.config.MavenMirrorsProperties.MirrorData>"
}
] }
},
{
"name": "mongodb.host",
"type": "java.lang.String",
"description": "MongoDB host."
},
{
"name": "mongodb.port",
"type": "java.lang.Integer",
"description": "MongoDB port."
},
{
"name": "mongodb.database",
"type": "java.lang.String",
"description": "MongoDB database name."
}
]
}

5
mirror-web/src/main/resources/application.yml
View File

@@ -15,6 +15,11 @@ minio:
secretKey: 'mirror123'
bucket: 'mirror'
mongodb:
host: 'dev.di9.ru'
port: 27017
database: 'mirror'
maven-mirrors:
list:
- id: 'maven_central'

1
versions.gradle
View File

@@ -3,6 +3,7 @@ ext {
slf4jVersion = '1.7.36'
minioVersion = '8.3.9'
springBootVerson = '2.6.6'
mongoDriver = '4.6.0'
// for tests only
junitJupiterVersion = '5.5.2'