отделили реализацию minio в отлдельный модуль

This commit is contained in:
2022-05-09 19:14:23 +03:00
parent 8e05d49198
commit f476c81f4e
18 changed files with 187 additions and 109 deletions

View File

@@ -1,34 +1,6 @@
apply plugin: 'java-library' apply plugin: 'java-library'
apply plugin: 'idea'
configurations {
integrationTestImplementation {
extendsFrom(testImplementation)
}
integrationTestRuntimeOnly {
extendsFrom(testRuntimeOnly)
}
}
sourceSets {
integrationTest {
java {
compileClasspath += main.output + test.output
runtimeClasspath += main.output + test.output
srcDir file('src/integrationTest/java')
}
resources {
srcDir file('src/integrationTest/resources')
}
}
}
idea.module {
testSourceDirs += sourceSets.integrationTest.java.srcDirs
testResourceDirs += sourceSets.integrationTest.resources.srcDirs
}
dependencies { dependencies {
api("io.minio:minio:${minioVersion}") api("com.squareup.okhttp3:okhttp:${okhttpVersion}")
api("org.mongodb:mongodb-driver-sync:${mongoDriver}") api("org.mongodb:mongodb-driver-sync:${mongoDriver}")
} }

View File

@@ -0,0 +1,10 @@
package ru.di9.mirror.core.domain;
public interface FileItem {
boolean isDir();
String name();
long size();
}

View File

@@ -1,28 +0,0 @@
package ru.di9.mirror.core.domain;
import io.minio.Result;
import io.minio.messages.Item;
import lombok.SneakyThrows;
public record ItemRecord(Result<Item> itemResult, String prefix) {
@SneakyThrows
public boolean isDir() {
return itemResult.get().isDir();
}
@SneakyThrows
public String name() {
String objectName = itemResult.get().objectName();
if (isDir()) {
return objectName.substring(prefix.length() - 1, objectName.length() - 1);
} else {
return objectName.substring(prefix.length() - 1);
}
}
@SneakyThrows
public long size() {
return itemResult.get().size();
}
}

View File

@@ -1,18 +1,18 @@
package ru.di9.mirror.core.handler; package ru.di9.mirror.core.handler;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import ru.di9.mirror.core.domain.ItemRecord; import ru.di9.mirror.core.domain.FileItem;
import ru.di9.mirror.core.service.MinioService; import ru.di9.mirror.core.repository.FileStorageRepository;
import java.util.List; import java.util.List;
@RequiredArgsConstructor @RequiredArgsConstructor
public class IndexOfHandler { public class IndexOfHandler {
private final MinioService minioService; private final FileStorageRepository fileStorageRepository;
public List<ItemRecord> walker(String path) { public List<FileItem> walker(String path) {
return minioService.list("/local/" + path); return fileStorageRepository.list("/local/" + path);
} }
} }

View File

@@ -8,7 +8,7 @@ import ru.di9.mirror.core.entity.ArtifactEntity;
import ru.di9.mirror.core.handler.response.GetFileResponse; import ru.di9.mirror.core.handler.response.GetFileResponse;
import ru.di9.mirror.core.repository.ArtifactRepository; import ru.di9.mirror.core.repository.ArtifactRepository;
import ru.di9.mirror.core.service.ExternalMavenService; import ru.di9.mirror.core.service.ExternalMavenService;
import ru.di9.mirror.core.service.MinioService; import ru.di9.mirror.core.repository.FileStorageRepository;
import java.io.InputStream; import java.io.InputStream;
import java.util.List; import java.util.List;
@@ -18,7 +18,7 @@ import java.util.Optional;
@RequiredArgsConstructor @RequiredArgsConstructor
public class MavenHandler { public class MavenHandler {
private final MinioService minioService; private final FileStorageRepository fileStorageRepository;
private final List<ExternalMavenService> externalMavenServices; private final List<ExternalMavenService> externalMavenServices;
private final ArtifactRepository repository; private final ArtifactRepository repository;
@@ -33,7 +33,7 @@ public class MavenHandler {
log.info(artifactRecord.toString()); log.info(artifactRecord.toString());
} }
Optional<InputStream> optionalInputStream = minioService.get("/local/" + path); Optional<InputStream> optionalInputStream = fileStorageRepository.findByName("/local/" + path);
if (optionalInputStream.isPresent()) { if (optionalInputStream.isPresent()) {
return optionalInputStream return optionalInputStream
.map(inputStream -> new GetFileResponse(fileInfo.name(), inputStream)); .map(inputStream -> new GetFileResponse(fileInfo.name(), inputStream));
@@ -47,7 +47,7 @@ public class MavenHandler {
for (ExternalMavenService externalMavenService : externalMavenServices) { for (ExternalMavenService externalMavenService : externalMavenServices) {
final String nameForStore = "/" + externalMavenService.getId() + "/" + path; final String nameForStore = "/" + externalMavenService.getId() + "/" + path;
result = minioService.get(nameForStore); result = fileStorageRepository.findByName(nameForStore);
if (result.isPresent()) { if (result.isPresent()) {
return result return result
.map(inputStream -> new GetFileResponse(fileName, inputStream)); .map(inputStream -> new GetFileResponse(fileName, inputStream));
@@ -58,8 +58,8 @@ public class MavenHandler {
repository.save(ArtifactEntity.fromRecord(artifactRecord)); repository.save(ArtifactEntity.fromRecord(artifactRecord));
} }
minioService.put(nameForStore, result.get()); fileStorageRepository.save(nameForStore, result.get());
return minioService.get(nameForStore) return fileStorageRepository.findByName(nameForStore)
.map(inputStream -> new GetFileResponse(fileName, inputStream)); .map(inputStream -> new GetFileResponse(fileName, inputStream));
} }
} }
@@ -78,6 +78,6 @@ public class MavenHandler {
log.info(artifactRecord.toString()); log.info(artifactRecord.toString());
} }
minioService.put("/local/" + path, inputStream); fileStorageRepository.save("/local/" + path, inputStream);
} }
} }

View File

@@ -0,0 +1,16 @@
package ru.di9.mirror.core.repository;
import ru.di9.mirror.core.domain.FileItem;
import java.io.InputStream;
import java.util.List;
import java.util.Optional;
public interface FileStorageRepository {
Optional<InputStream> findByName(String name);
void save(String name, InputStream inputStream);
List<FileItem> list(String prefix);
}

35
mirror-minio/build.gradle Normal file
View File

@@ -0,0 +1,35 @@
apply plugin: 'java-library'
apply plugin: 'idea'
configurations {
integrationTestImplementation {
extendsFrom(testImplementation)
}
integrationTestRuntimeOnly {
extendsFrom(testRuntimeOnly)
}
}
sourceSets {
integrationTest {
java {
compileClasspath += main.output + test.output
runtimeClasspath += main.output + test.output
srcDir file('src/integrationTest/java')
}
resources {
srcDir file('src/integrationTest/resources')
}
}
}
idea.module {
testSourceDirs += sourceSets.integrationTest.java.srcDirs
testResourceDirs += sourceSets.integrationTest.resources.srcDirs
}
dependencies {
implementation(project(':mirror-core'))
api("io.minio:minio:${minioVersion}")
}

View File

@@ -4,7 +4,8 @@ import io.minio.MinioClient;
import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import ru.di9.mirror.core.domain.ItemRecord; import ru.di9.mirror.core.domain.FileItem;
import ru.di9.mirror.minio.service.MinioRepository;
import java.io.InputStream; import java.io.InputStream;
import java.util.List; import java.util.List;
@@ -18,7 +19,7 @@ class MinioServiceIntTest {
static final String secretKey = "mirror123"; static final String secretKey = "mirror123";
static final String bucket = "mirror"; static final String bucket = "mirror";
static MinioClient minioClient; static MinioClient minioClient;
MinioService minioService; MinioRepository minioService;
@BeforeAll @BeforeAll
static void beforeAll() { static void beforeAll() {
@@ -30,12 +31,12 @@ class MinioServiceIntTest {
@BeforeEach @BeforeEach
void setUp() { void setUp() {
minioService = new MinioService(minioClient, bucket); minioService = new MinioRepository(minioClient, bucket);
} }
@Test @Test
void list() { void list() {
List<ItemRecord> list = minioService.list("/local/ghast/ghast-tools/"); List<FileItem> list = minioService.list("/local/ghast/ghast-tools/");
assertNotNull(list); assertNotNull(list);
assertFalse(list.isEmpty()); assertFalse(list.isEmpty());
@@ -52,7 +53,7 @@ class MinioServiceIntTest {
@Test @Test
void notExists() { void notExists() {
Optional<InputStream> optional = minioService.get("/not/exists/path/to/object"); Optional<InputStream> optional = minioService.findByName("/not/exists/path/to/object");
assertNotNull(optional); assertNotNull(optional);
assertTrue(optional.isEmpty()); assertTrue(optional.isEmpty());
@@ -60,7 +61,7 @@ class MinioServiceIntTest {
@Test @Test
void exists() { void exists() {
Optional<InputStream> optional = minioService.get("/local/ghast/ghast-tools/maven-metadata.xml"); Optional<InputStream> optional = minioService.findByName("/local/ghast/ghast-tools/maven-metadata.xml");
assertNotNull(optional); assertNotNull(optional);
assertTrue(optional.isPresent()); assertTrue(optional.isPresent());

View File

@@ -0,0 +1,51 @@
package ru.di9.mirror.minio.domain;
import io.minio.Result;
import io.minio.messages.Item;
import lombok.RequiredArgsConstructor;
import ru.di9.mirror.core.domain.FileItem;
import ru.di9.mirror.minio.exception.MinioException;
@RequiredArgsConstructor
public class MinioFileItem implements FileItem {
private final Result<Item> itemResult;
private final String prefix;
@Override
public boolean isDir() {
try {
return itemResult.get().isDir();
} catch (Exception e) {
throw createException(e);
}
}
@Override
public String name() {
try {
String objectName = itemResult.get().objectName();
if (isDir()) {
return objectName.substring(prefix.length() - 1, objectName.length() - 1);
} else {
return objectName.substring(prefix.length() - 1);
}
} catch (MinioException e) {
throw e;
} catch (Exception e) {
throw createException(e);
}
}
@Override
public long size() {
try {
return itemResult.get().size();
} catch (Exception e) {
throw createException(e);
}
}
private MinioException createException(Exception e) {
return new MinioException("Error get result item", e);
}
}

View File

@@ -0,0 +1,7 @@
package ru.di9.mirror.minio.exception;
public class MinioException extends RuntimeException {
public MinioException(String message, Throwable cause) {
super(message, cause);
}
}

View File

@@ -1,11 +1,13 @@
package ru.di9.mirror.core.service; package ru.di9.mirror.minio.service;
import io.minio.*; import io.minio.*;
import io.minio.errors.*; import io.minio.errors.ErrorResponseException;
import io.minio.messages.Item; import io.minio.messages.Item;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows; import ru.di9.mirror.core.domain.FileItem;
import ru.di9.mirror.core.domain.ItemRecord; import ru.di9.mirror.core.repository.FileStorageRepository;
import ru.di9.mirror.minio.domain.MinioFileItem;
import ru.di9.mirror.minio.exception.MinioException;
import java.io.InputStream; import java.io.InputStream;
import java.util.ArrayList; import java.util.ArrayList;
@@ -13,24 +15,28 @@ import java.util.List;
import java.util.Optional; import java.util.Optional;
@RequiredArgsConstructor @RequiredArgsConstructor
public class MinioService { public class MinioRepository implements FileStorageRepository {
private static final long UNKNOWN_SIZE = -1L; private static final long UNKNOWN_SIZE = -1L;
private static final long PART_SIZE = (5 * 1024 * 1024); // 5 MB private static final long PART_SIZE = (5 * 1024 * 1024); // 5 MB
private final MinioClient minioClient; private final MinioClient minioClient;
private final String bucket; private final String bucket;
@SneakyThrows @Override
public void put(String name, InputStream inputStream) { public void save(String name, InputStream inputStream) {
try {
minioClient.putObject(PutObjectArgs.builder() minioClient.putObject(PutObjectArgs.builder()
.bucket(bucket) .bucket(bucket)
.object(name) .object(name)
.stream(inputStream, UNKNOWN_SIZE, PART_SIZE) .stream(inputStream, UNKNOWN_SIZE, PART_SIZE)
.build()); .build());
} catch (Exception e) {
throw new MinioException("Error put file", e);
}
} }
@SneakyThrows @Override
public Optional<InputStream> get(String name) { public Optional<InputStream> findByName(String name) {
try { try {
return Optional.of(minioClient.getObject(GetObjectArgs.builder() return Optional.of(minioClient.getObject(GetObjectArgs.builder()
.bucket(bucket) .bucket(bucket)
@@ -40,12 +46,15 @@ public class MinioService {
if (e.errorResponse().code().equalsIgnoreCase("NoSuchKey")) { if (e.errorResponse().code().equalsIgnoreCase("NoSuchKey")) {
return Optional.empty(); return Optional.empty();
} else { } else {
throw e; throw new MinioException("Error get file", e);
} }
} catch (Exception e) {
throw new MinioException("Error get file", e);
} }
} }
public List<ItemRecord> list(String prefix) { @Override
public List<FileItem> list(String prefix) {
if (!prefix.endsWith("/")) { if (!prefix.endsWith("/")) {
prefix = prefix + "/"; prefix = prefix + "/";
} }
@@ -55,9 +64,9 @@ public class MinioService {
.prefix(prefix) .prefix(prefix)
.build()); .build());
List<ItemRecord> list = new ArrayList<>(); List<FileItem> list = new ArrayList<>();
for (Result<Item> result : results) { for (Result<Item> result : results) {
list.add(new ItemRecord(result, prefix)); list.add(new MinioFileItem(result, prefix));
} }
return list; return list;

View File

@@ -7,7 +7,8 @@ import io.minio.messages.Item;
import lombok.SneakyThrows; import lombok.SneakyThrows;
import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import ru.di9.mirror.core.domain.ItemRecord; import ru.di9.mirror.core.domain.FileItem;
import ru.di9.mirror.minio.service.MinioRepository;
import java.util.List; import java.util.List;
@@ -31,8 +32,8 @@ class MinioServiceTest {
@Test @Test
void list() { void list() {
MinioService minioService = new MinioService(mockMinioClient, bucket); MinioRepository minioService = new MinioRepository(mockMinioClient, bucket);
List<ItemRecord> list = minioService.list("/ghast/ghast-tools/"); List<FileItem> list = minioService.list("/ghast/ghast-tools/");
assertNotNull(list); assertNotNull(list);
assertFalse(list.isEmpty()); assertFalse(list.isEmpty());

View File

@@ -3,6 +3,7 @@ apply plugin: 'org.springframework.boot'
dependencies { dependencies {
implementation(project(':mirror-core')) implementation(project(':mirror-core'))
implementation(project(':mirror-minio'))
annotationProcessor("org.springframework.boot:spring-boot-configuration-processor:${springBootVerson}") annotationProcessor("org.springframework.boot:spring-boot-configuration-processor:${springBootVerson}")
implementation("org.springframework.boot:spring-boot-starter-web:${springBootVerson}") implementation("org.springframework.boot:spring-boot-starter-web:${springBootVerson}")

View File

@@ -4,14 +4,15 @@ import io.minio.MinioClient;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Configuration;
import ru.di9.mirror.core.service.MinioService; import ru.di9.mirror.core.repository.FileStorageRepository;
import ru.di9.mirror.minio.service.MinioRepository;
@Configuration @Configuration
public class MinioConfig { public class MinioConfig {
@Bean @Bean
public MinioService minioService(MinioClient minioClient, @Value("${minio.bucket}") String bucket) { public FileStorageRepository fileStorageRepository(MinioClient minioClient, @Value("${minio.bucket}") String bucket) {
return new MinioService(minioClient, bucket); return new MinioRepository(minioClient, bucket);
} }
@Bean @Bean

View File

@@ -9,7 +9,7 @@ import ru.di9.mirror.core.handler.IndexOfHandler;
import ru.di9.mirror.core.handler.MavenHandler; import ru.di9.mirror.core.handler.MavenHandler;
import ru.di9.mirror.core.repository.ArtifactRepository; import ru.di9.mirror.core.repository.ArtifactRepository;
import ru.di9.mirror.core.service.ExternalMavenService; import ru.di9.mirror.core.service.ExternalMavenService;
import ru.di9.mirror.core.service.MinioService; import ru.di9.mirror.core.repository.FileStorageRepository;
import java.util.List; import java.util.List;
@@ -17,15 +17,15 @@ import java.util.List;
public class WebConfig { public class WebConfig {
@Bean @Bean
public MavenHandler mavenHandler(MinioService minioService, public MavenHandler mavenHandler(FileStorageRepository fileStorageRepository,
List<ExternalMavenService> externalMavenServices, List<ExternalMavenService> externalMavenServices,
ArtifactRepository artifactRepository) { ArtifactRepository artifactRepository) {
return new MavenHandler(minioService, externalMavenServices, artifactRepository); return new MavenHandler(fileStorageRepository, externalMavenServices, artifactRepository);
} }
@Bean @Bean
public IndexOfHandler indexOfHandler(MinioService minioService) { public IndexOfHandler indexOfHandler(FileStorageRepository fileStorageRepository) {
return new IndexOfHandler(minioService); return new IndexOfHandler(fileStorageRepository);
} }
@Bean @Bean

View File

@@ -7,7 +7,7 @@ import org.springframework.web.bind.annotation.ModelAttribute;
import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
import ru.di9.mirror.core.Utils; import ru.di9.mirror.core.Utils;
import ru.di9.mirror.core.domain.ItemRecord; import ru.di9.mirror.core.domain.FileItem;
import ru.di9.mirror.core.handler.IndexOfHandler; import ru.di9.mirror.core.handler.IndexOfHandler;
import java.util.ArrayList; import java.util.ArrayList;
@@ -28,7 +28,7 @@ public class IndexOfMavenController {
@ModelAttribute("model") ModelMap model) { @ModelAttribute("model") ModelMap model) {
String path = correctingHttpPath(httpPath); String path = correctingHttpPath(httpPath);
List<ItemRecord> walker = indexOfHandler.walker(path); List<FileItem> walker = indexOfHandler.walker(path);
List<ModelLink> links = new ArrayList<>(); List<ModelLink> links = new ArrayList<>();
StringBuilder sb = new StringBuilder("/maven"); StringBuilder sb = new StringBuilder("/maven");
@@ -46,13 +46,13 @@ public class IndexOfMavenController {
sb.setLength(resetLength); sb.setLength(resetLength);
sb.append("/"); sb.append("/");
resetLength = sb.length(); resetLength = sb.length();
for (ItemRecord itemRecord : walker) { for (FileItem fileItem : walker) {
if (Utils.isNotEmptyString(path)) { if (Utils.isNotEmptyString(path)) {
sb.append(path).append("/"); sb.append(path).append("/");
} }
sb.append(itemRecord.name()); sb.append(fileItem.name());
links.add(new ModelLink(itemRecord.name() + (itemRecord.isDir() ? "/" : ""), sb.toString())); links.add(new ModelLink(fileItem.name() + (fileItem.isDir() ? "/" : ""), sb.toString()));
sb.setLength(resetLength); sb.setLength(resetLength);
} }

View File

@@ -1,4 +1,5 @@
rootProject.name = 'project-mirror' rootProject.name = 'project-mirror'
include('mirror-core') include('mirror-core')
include('mirror-minio')
include('mirror-web') include('mirror-web')

View File

@@ -4,6 +4,7 @@ ext {
minioVersion = '8.3.9' minioVersion = '8.3.9'
springBootVerson = '2.6.6' springBootVerson = '2.6.6'
mongoDriver = '4.6.0' mongoDriver = '4.6.0'
okhttpVersion = '4.8.1'
// for tests only // for tests only
junitJupiterVersion = '5.5.2' junitJupiterVersion = '5.5.2'