Voomra/Project-Mirror
0
Fork 0
Code Issues 2 Releases Activity

отделили реализацию minio в отлдельный модуль

Browse Source
This commit is contained in:
DmitriyMX
2022-05-09 19:14:23 +03:00
parent 8e05d49198
commit f476c81f4e
18 changed files with 187 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
mirror-core/build.gradle
View File

@@ -1,34 +1,6 @@
apply plugin: 'java-library'
apply plugin: 'idea'
configurations {
integrationTestImplementation {
extendsFrom(testImplementation)
}
integrationTestRuntimeOnly {
extendsFrom(testRuntimeOnly)
}
}
sourceSets {
integrationTest {
java {
compileClasspath += main.output + test.output
runtimeClasspath += main.output + test.output
srcDir file('src/integrationTest/java')
}
resources {
srcDir file('src/integrationTest/resources')
}
}
}
idea.module {
testSourceDirs += sourceSets.integrationTest.java.srcDirs
testResourceDirs += sourceSets.integrationTest.resources.srcDirs
}
dependencies {
api("io.minio:minio:${minioVersion}")
api("com.squareup.okhttp3:okhttp:${okhttpVersion}")
api("org.mongodb:mongodb-driver-sync:${mongoDriver}")
}

10
mirror-core/src/main/java/ru/di9/mirror/core/domain/FileItem.java Normal file
View File

@@ -0,0 +1,10 @@
package ru.di9.mirror.core.domain;
public interface FileItem {
boolean isDir();
String name();
long size();
}

28
mirror-core/src/main/java/ru/di9/mirror/core/domain/ItemRecord.java
View File

@@ -1,28 +0,0 @@
package ru.di9.mirror.core.domain;
import io.minio.Result;
import io.minio.messages.Item;
import lombok.SneakyThrows;
public record ItemRecord(Result<Item> itemResult, String prefix) {
@SneakyThrows
public boolean isDir() {
return itemResult.get().isDir();
}
@SneakyThrows
public String name() {
String objectName = itemResult.get().objectName();
if (isDir()) {
return objectName.substring(prefix.length() - 1, objectName.length() - 1);
} else {
return objectName.substring(prefix.length() - 1);
}
}
@SneakyThrows
public long size() {
return itemResult.get().size();
}
}

10
mirror-core/src/main/java/ru/di9/mirror/core/handler/IndexOfHandler.java
View File

@@ -1,18 +1,18 @@
package ru.di9.mirror.core.handler;
import lombok.RequiredArgsConstructor;
import ru.di9.mirror.core.domain.ItemRecord;
import ru.di9.mirror.core.service.MinioService;
import ru.di9.mirror.core.domain.FileItem;
import ru.di9.mirror.core.repository.FileStorageRepository;
import java.util.List;
@RequiredArgsConstructor
public class IndexOfHandler {
private final MinioService minioService;
private final FileStorageRepository fileStorageRepository;
public List<ItemRecord> walker(String path) {
return minioService.list("/local/" + path);
public List<FileItem> walker(String path) {
return fileStorageRepository.list("/local/" + path);
}
}

14
mirror-core/src/main/java/ru/di9/mirror/core/handler/MavenHandler.java
View File

@@ -8,7 +8,7 @@ import ru.di9.mirror.core.entity.ArtifactEntity;
import ru.di9.mirror.core.handler.response.GetFileResponse;
import ru.di9.mirror.core.repository.ArtifactRepository;
import ru.di9.mirror.core.service.ExternalMavenService;
import ru.di9.mirror.core.service.MinioService;
import ru.di9.mirror.core.repository.FileStorageRepository;
import java.io.InputStream;
import java.util.List;
@@ -18,7 +18,7 @@ import java.util.Optional;
@RequiredArgsConstructor
public class MavenHandler {
private final MinioService minioService;
private final FileStorageRepository fileStorageRepository;
private final List<ExternalMavenService> externalMavenServices;
private final ArtifactRepository repository;
@@ -33,7 +33,7 @@ public class MavenHandler {
log.info(artifactRecord.toString());
}
Optional<InputStream> optionalInputStream = minioService.get("/local/" + path);
Optional<InputStream> optionalInputStream = fileStorageRepository.findByName("/local/" + path);
if (optionalInputStream.isPresent()) {
return optionalInputStream
.map(inputStream -> new GetFileResponse(fileInfo.name(), inputStream));
@@ -47,7 +47,7 @@ public class MavenHandler {
for (ExternalMavenService externalMavenService : externalMavenServices) {
final String nameForStore = "/" + externalMavenService.getId() + "/" + path;
result = minioService.get(nameForStore);
result = fileStorageRepository.findByName(nameForStore);
if (result.isPresent()) {
return result
.map(inputStream -> new GetFileResponse(fileName, inputStream));
@@ -58,8 +58,8 @@ public class MavenHandler {
repository.save(ArtifactEntity.fromRecord(artifactRecord));
}
minioService.put(nameForStore, result.get());
return minioService.get(nameForStore)
fileStorageRepository.save(nameForStore, result.get());
return fileStorageRepository.findByName(nameForStore)
.map(inputStream -> new GetFileResponse(fileName, inputStream));
}
}
@@ -78,6 +78,6 @@ public class MavenHandler {
log.info(artifactRecord.toString());
}
minioService.put("/local/" + path, inputStream);
fileStorageRepository.save("/local/" + path, inputStream);
}
}

16
mirror-core/src/main/java/ru/di9/mirror/core/repository/FileStorageRepository.java Normal file
View File

@@ -0,0 +1,16 @@
package ru.di9.mirror.core.repository;
import ru.di9.mirror.core.domain.FileItem;
import java.io.InputStream;
import java.util.List;
import java.util.Optional;
public interface FileStorageRepository {
Optional<InputStream> findByName(String name);
void save(String name, InputStream inputStream);
List<FileItem> list(String prefix);
}

35
mirror-minio/build.gradle Normal file
View File

@@ -0,0 +1,35 @@
apply plugin: 'java-library'
apply plugin: 'idea'
configurations {
integrationTestImplementation {
extendsFrom(testImplementation)
}
integrationTestRuntimeOnly {
extendsFrom(testRuntimeOnly)
}
}
sourceSets {
integrationTest {
java {
compileClasspath += main.output + test.output
runtimeClasspath += main.output + test.output
srcDir file('src/integrationTest/java')
}
resources {
srcDir file('src/integrationTest/resources')
}
}
}
idea.module {
testSourceDirs += sourceSets.integrationTest.java.srcDirs
testResourceDirs += sourceSets.integrationTest.resources.srcDirs
}
dependencies {
implementation(project(':mirror-core'))
api("io.minio:minio:${minioVersion}")
}

13
mirror-core/src/integrationTest/java/ru/di9/mirror/core/service/MinioServiceIntTest.java → mirror-minio/src/integrationTest/java/ru/di9/mirror/core/service/MinioServiceIntTest.java
View File

@@ -4,7 +4,8 @@ import io.minio.MinioClient;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import ru.di9.mirror.core.domain.ItemRecord;
import ru.di9.mirror.core.domain.FileItem;
import ru.di9.mirror.minio.service.MinioRepository;
import java.io.InputStream;
import java.util.List;
@@ -18,7 +19,7 @@ class MinioServiceIntTest {
static final String secretKey = "mirror123";
static final String bucket = "mirror";
static MinioClient minioClient;
MinioService minioService;
MinioRepository minioService;
@BeforeAll
static void beforeAll() {
@@ -30,12 +31,12 @@ class MinioServiceIntTest {
@BeforeEach
void setUp() {
minioService = new MinioService(minioClient, bucket);
minioService = new MinioRepository(minioClient, bucket);
}
@Test
void list() {
List<ItemRecord> list = minioService.list("/local/ghast/ghast-tools/");
List<FileItem> list = minioService.list("/local/ghast/ghast-tools/");
assertNotNull(list);
assertFalse(list.isEmpty());
@@ -52,7 +53,7 @@ class MinioServiceIntTest {
@Test
void notExists() {
Optional<InputStream> optional = minioService.get("/not/exists/path/to/object");
Optional<InputStream> optional = minioService.findByName("/not/exists/path/to/object");
assertNotNull(optional);
assertTrue(optional.isEmpty());
@@ -60,7 +61,7 @@ class MinioServiceIntTest {
@Test
void exists() {
Optional<InputStream> optional = minioService.get("/local/ghast/ghast-tools/maven-metadata.xml");
Optional<InputStream> optional = minioService.findByName("/local/ghast/ghast-tools/maven-metadata.xml");
assertNotNull(optional);
assertTrue(optional.isPresent());

51
mirror-minio/src/main/java/ru/di9/mirror/minio/domain/MinioFileItem.java Normal file
View File

@@ -0,0 +1,51 @@
package ru.di9.mirror.minio.domain;
import io.minio.Result;
import io.minio.messages.Item;
import lombok.RequiredArgsConstructor;
import ru.di9.mirror.core.domain.FileItem;
import ru.di9.mirror.minio.exception.MinioException;
@RequiredArgsConstructor
public class MinioFileItem implements FileItem {
private final Result<Item> itemResult;
private final String prefix;
@Override
public boolean isDir() {
try {
return itemResult.get().isDir();
} catch (Exception e) {
throw createException(e);
}
}
@Override
public String name() {
try {
String objectName = itemResult.get().objectName();
if (isDir()) {
return objectName.substring(prefix.length() - 1, objectName.length() - 1);
} else {
return objectName.substring(prefix.length() - 1);
}
} catch (MinioException e) {
throw e;
} catch (Exception e) {
throw createException(e);
}
}
@Override
public long size() {
try {
return itemResult.get().size();
} catch (Exception e) {
throw createException(e);
}
}
private MinioException createException(Exception e) {
return new MinioException("Error get result item", e);
}
}

7
mirror-minio/src/main/java/ru/di9/mirror/minio/exception/MinioException.java Normal file
View File

@@ -0,0 +1,7 @@
package ru.di9.mirror.minio.exception;
public class MinioException extends RuntimeException {
public MinioException(String message, Throwable cause) {
super(message, cause);
}
}

45
mirror-core/src/main/java/ru/di9/mirror/core/service/MinioService.java → mirror-minio/src/main/java/ru/di9/mirror/minio/service/MinioRepository.java
View File

@@ -1,11 +1,13 @@
package ru.di9.mirror.core.service;
package ru.di9.mirror.minio.service;
import io.minio.*;
import io.minio.errors.*;
import io.minio.errors.ErrorResponseException;
import io.minio.messages.Item;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import ru.di9.mirror.core.domain.ItemRecord;
import ru.di9.mirror.core.domain.FileItem;
import ru.di9.mirror.core.repository.FileStorageRepository;
import ru.di9.mirror.minio.domain.MinioFileItem;
import ru.di9.mirror.minio.exception.MinioException;
import java.io.InputStream;
import java.util.ArrayList;
@@ -13,24 +15,28 @@ import java.util.List;
import java.util.Optional;
@RequiredArgsConstructor
public class MinioService {
public class MinioRepository implements FileStorageRepository {
private static final long UNKNOWN_SIZE = -1L;
private static final long PART_SIZE = (5 * 1024 * 1024); // 5 MB
private final MinioClient minioClient;
private final String bucket;
@SneakyThrows
public void put(String name, InputStream inputStream) {
minioClient.putObject(PutObjectArgs.builder()
.bucket(bucket)
.object(name)
.stream(inputStream, UNKNOWN_SIZE, PART_SIZE)
.build());
@Override
public void save(String name, InputStream inputStream) {
try {
minioClient.putObject(PutObjectArgs.builder()
.bucket(bucket)
.object(name)
.stream(inputStream, UNKNOWN_SIZE, PART_SIZE)
.build());
} catch (Exception e) {
throw new MinioException("Error put file", e);
}
}
@SneakyThrows
public Optional<InputStream> get(String name) {
@Override
public Optional<InputStream> findByName(String name) {
try {
return Optional.of(minioClient.getObject(GetObjectArgs.builder()
.bucket(bucket)
@@ -40,12 +46,15 @@ public class MinioService {
if (e.errorResponse().code().equalsIgnoreCase("NoSuchKey")) {
return Optional.empty();
} else {
throw e;
throw new MinioException("Error get file", e);
}
} catch (Exception e) {
throw new MinioException("Error get file", e);
}
}
public List<ItemRecord> list(String prefix) {
@Override
public List<FileItem> list(String prefix) {
if (!prefix.endsWith("/")) {
prefix = prefix + "/";
}
@@ -55,9 +64,9 @@ public class MinioService {
.prefix(prefix)
.build());
List<ItemRecord> list = new ArrayList<>();
List<FileItem> list = new ArrayList<>();
for (Result<Item> result : results) {
list.add(new ItemRecord(result, prefix));
list.add(new MinioFileItem(result, prefix));
}
return list;

7
mirror-core/src/test/java/ru/di9/mirror/core/service/MinioServiceTest.java → mirror-minio/src/test/java/ru/di9/mirror/core/service/MinioServiceTest.java
View File

@@ -7,7 +7,8 @@ import io.minio.messages.Item;
import lombok.SneakyThrows;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import ru.di9.mirror.core.domain.ItemRecord;
import ru.di9.mirror.core.domain.FileItem;
import ru.di9.mirror.minio.service.MinioRepository;
import java.util.List;
@@ -31,8 +32,8 @@ class MinioServiceTest {
@Test
void list() {
MinioService minioService = new MinioService(mockMinioClient, bucket);
List<ItemRecord> list = minioService.list("/ghast/ghast-tools/");
MinioRepository minioService = new MinioRepository(mockMinioClient, bucket);
List<FileItem> list = minioService.list("/ghast/ghast-tools/");
assertNotNull(list);
assertFalse(list.isEmpty());

1
mirror-web/build.gradle
View File

@@ -3,6 +3,7 @@ apply plugin: 'org.springframework.boot'
dependencies {
implementation(project(':mirror-core'))
implementation(project(':mirror-minio'))
annotationProcessor("org.springframework.boot:spring-boot-configuration-processor:${springBootVerson}")
implementation("org.springframework.boot:spring-boot-starter-web:${springBootVerson}")

7
mirror-web/src/main/java/ru/di9/mirror/web/config/MinioConfig.java
View File

@@ -4,14 +4,15 @@ import io.minio.MinioClient;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import ru.di9.mirror.core.service.MinioService;
import ru.di9.mirror.core.repository.FileStorageRepository;
import ru.di9.mirror.minio.service.MinioRepository;
@Configuration
public class MinioConfig {
@Bean
public MinioService minioService(MinioClient minioClient, @Value("${minio.bucket}") String bucket) {
return new MinioService(minioClient, bucket);
public FileStorageRepository fileStorageRepository(MinioClient minioClient, @Value("${minio.bucket}") String bucket) {
return new MinioRepository(minioClient, bucket);
}
@Bean

10
mirror-web/src/main/java/ru/di9/mirror/web/config/WebConfig.java
View File

@@ -9,7 +9,7 @@ import ru.di9.mirror.core.handler.IndexOfHandler;
import ru.di9.mirror.core.handler.MavenHandler;
import ru.di9.mirror.core.repository.ArtifactRepository;
import ru.di9.mirror.core.service.ExternalMavenService;
import ru.di9.mirror.core.service.MinioService;
import ru.di9.mirror.core.repository.FileStorageRepository;
import java.util.List;
@@ -17,15 +17,15 @@ import java.util.List;
public class WebConfig {
@Bean
public MavenHandler mavenHandler(MinioService minioService,
public MavenHandler mavenHandler(FileStorageRepository fileStorageRepository,
List<ExternalMavenService> externalMavenServices,
ArtifactRepository artifactRepository) {
return new MavenHandler(minioService, externalMavenServices, artifactRepository);
return new MavenHandler(fileStorageRepository, externalMavenServices, artifactRepository);
}
@Bean
public IndexOfHandler indexOfHandler(MinioService minioService) {
return new IndexOfHandler(minioService);
public IndexOfHandler indexOfHandler(FileStorageRepository fileStorageRepository) {
return new IndexOfHandler(fileStorageRepository);
}
@Bean

10
mirror-web/src/main/java/ru/di9/mirror/web/controller/IndexOfMavenController.java
View File

@@ -7,7 +7,7 @@ import org.springframework.web.bind.annotation.ModelAttribute;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import ru.di9.mirror.core.Utils;
import ru.di9.mirror.core.domain.ItemRecord;
import ru.di9.mirror.core.domain.FileItem;
import ru.di9.mirror.core.handler.IndexOfHandler;
import java.util.ArrayList;
@@ -28,7 +28,7 @@ public class IndexOfMavenController {
@ModelAttribute("model") ModelMap model) {
String path = correctingHttpPath(httpPath);
List<ItemRecord> walker = indexOfHandler.walker(path);
List<FileItem> walker = indexOfHandler.walker(path);
List<ModelLink> links = new ArrayList<>();
StringBuilder sb = new StringBuilder("/maven");
@@ -46,13 +46,13 @@ public class IndexOfMavenController {
sb.setLength(resetLength);
sb.append("/");
resetLength = sb.length();
for (ItemRecord itemRecord : walker) {
for (FileItem fileItem : walker) {
if (Utils.isNotEmptyString(path)) {
sb.append(path).append("/");
}
sb.append(itemRecord.name());
sb.append(fileItem.name());
links.add(new ModelLink(itemRecord.name() + (itemRecord.isDir() ? "/" : ""), sb.toString()));
links.add(new ModelLink(fileItem.name() + (fileItem.isDir() ? "/" : ""), sb.toString()));
sb.setLength(resetLength);
}

1
settings.gradle
View File

@@ -1,4 +1,5 @@
rootProject.name = 'project-mirror'
include('mirror-core')
include('mirror-minio')
include('mirror-web')

1
versions.gradle
View File

@@ -4,6 +4,7 @@ ext {
minioVersion = '8.3.9'
springBootVerson = '2.6.6'
mongoDriver = '4.6.0'
okhttpVersion = '4.8.1'
// for tests only
junitJupiterVersion = '5.5.2'