chore(prod): sync with main #4

Merged
dylan merged 67 commits from master into prod 2026-06-25 23:28:52 +01:00
12 changed files with 143 additions and 210 deletions
Showing only changes of commit 5651aa4ce9 - Show all commits

View File

@@ -8,8 +8,6 @@ services:
POSTGRES_DB: ${POSTGRES_DB}
volumes:
- ${POSTGRES_DATA}:/var/lib/postgresql/data
ports:
- "5432:5432"
backend:
build:

View File

@@ -15,12 +15,6 @@ RUN apt-get update && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
ENV POSTGRES_DATA=/app/db
ENV MEDIA_DATA=/app/data/media
ENV STREAM_DATA=/app/data/media/streams
VOLUME ["app/db", "app/data/media/streams", "/app/data/media"]
WORKDIR /app
COPY --from=builder /app/target/*.jar app.jar
EXPOSE 8080

View File

@@ -2,7 +2,7 @@ package com.ddf.vodsystem.dto;
import lombok.Data;
import java.io.File;
import java.nio.file.Path;
import java.time.Instant;
@Data
@@ -12,7 +12,7 @@ public class Job {
private JobState state = JobState.READY;
private String errorOutput;
private File download;
private Path download;
private Instant createdAt = Instant.now();
public Job(String uuid) {

View File

@@ -7,6 +7,7 @@ import com.ddf.vodsystem.entities.*;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Optional;
@@ -200,8 +201,8 @@ public class ClipService {
*/
public Clip saveClip(ClipOptions options,
User user,
String videoPath,
String thumbnailPath) {
Path videoPath,
Path thumbnailPath) {
Clip clip = new Clip();
clip.setUser(user);
clip.setTitle(options.getTitle() != null ? options.getTitle() : "Untitled Clip");
@@ -212,23 +213,23 @@ public class ClipService {
clip.setFps(options.getFps());
clip.setDuration(options.getDuration() - options.getStartPoint());
clip.setFileSize(options.getFileSize());
clip.setVideoPath(videoPath);
clip.setThumbnailPath(thumbnailPath);
clip.setVideoPath(videoPath.toString());
clip.setThumbnailPath(thumbnailPath.toString());
return clipRepository.save(clip);
}
public void persistClip(String title,
String description,
User user,
File clipFile,
Path clipFile,
String fileName) {
File newClipFile;
File thumbnailFile;
Path newClipFile;
Path thumbnailFile;
// Move temp file from temp dir to output dir
try {
newClipFile = directoryService.getUserClipsFile(user.getId(), fileName);
thumbnailFile = directoryService.getUserThumbnailsFile(user.getId(), fileName + ".png");
newClipFile = directoryService.getClipsDir(user.getId()).resolve(fileName);
thumbnailFile = directoryService.getThumbnailsDir(user.getId()).resolve(fileName + ".png");
directoryService.copyFile(clipFile, newClipFile);
} catch (IOException e) {
throw new StorageException("Failed to move clip from temporary directory to output directory", e);
@@ -255,7 +256,7 @@ public class ClipService {
clipMetadata.setTitle(title);
clipMetadata.setDescription(description);
Clip clip = saveClip(clipMetadata, user, newClipFile.getAbsolutePath(), thumbnailFile.getAbsolutePath());
Clip clip = saveClip(clipMetadata, user, newClipFile, thumbnailFile);
logger.info("Clip created successfully with ID: {}", clip.getId());
}

View File

@@ -10,7 +10,7 @@ import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.stream.Stream;
import java.nio.file.StandardCopyOption;
import org.slf4j.Logger;
@@ -20,135 +20,108 @@ public class DirectoryService {
private static final Logger logger = org.slf4j.LoggerFactory.getLogger(DirectoryService.class);
@Value("${storage.outputs}")
private String outputDir;
@Value("${media.users}")
private String usersDir;
@Value("${STREAM_DATA}")
private String streamDir;
@Value("${media.streams}")
private String streamsDir;
@Value("${storage.temp.inputs}")
@Value("${media.temp.inputs}")
private String tempInputsDir;
@Value("${storage.temp.outputs}")
@Value("${media.temp.outputs}")
private String tempOutputsDir;
private static final long TEMP_DIR_TIME_LIMIT = 3 * 60 * 60 * (long) 1000; // 3 hours
private static final long TEMP_DIR_CLEANUP_RATE = 30 * 60 * (long) 1000; // 30 minutes
public File getTempInputFile(String filename) {
String dir = tempInputsDir + File.separator + filename;
return new File(dir);
private static final String VODS_DIR_NAME = "vods";
private static final String CLIPS_DIR_NAME = "clips";
private static final String THUMBNAIL_DIR_NAME = "thumbnails";
public Path getTempInputDir() {
return Path.of(tempInputsDir);
}
public File getTempOutputFile(String filename) {
String dir = tempOutputsDir + File.separator + filename;
return new File(dir);
public Path getTempOutputDir() {
return Path.of(tempOutputsDir);
}
public File getUserFolder(Long userId) throws IOException {
if (userId == null) {
throw new IllegalArgumentException("User ID cannot be null or empty");
}
String dir = outputDir + File.separator + userId;
createDirectory(dir);
return new File(dir);
public Path getUserDir(Long userId) throws IOException {
Path userDir = Path.of(usersDir + File.separator + userId);
Files.createDirectories(userDir);
return userDir;
}
public File getUserClipsFile(Long userId, String filename) throws IOException {
if (filename == null || filename.isEmpty()) {
throw new IllegalArgumentException("File name cannot be null or empty");
}
File userDir = getUserFolder(userId);
String fullPath = userDir.getAbsolutePath() + File.separator + "clips" + File.separator + filename;
File file = new File(fullPath);
createDirectory(file.getParent());
return file;
public Path getVodsDir(Long userId) throws IOException {
Path vodDir = getUserDir(userId).resolve(VODS_DIR_NAME);
Files.createDirectories(vodDir);
return vodDir;
}
public File getVodFile(Long userId, String filename) throws IOException {
if (filename == null || filename.isEmpty()) {
throw new IllegalArgumentException("File name cannot be null or empty");
}
String userDir = getUserFolder(userId).getAbsolutePath();
File file = new File(userDir + File.separator + "vods" + File.separator + filename);
createDirectory(file.getParent());
return file;
public Path getClipsDir(Long userId) throws IOException {
Path clipsDir = getUserDir(userId).resolve(CLIPS_DIR_NAME);
Files.createDirectories(clipsDir);
return clipsDir;
}
public File getStreamDir(String streamKey) throws IOException {
File file = new File(streamDir + File.separator + streamKey);
createDirectory(file.getAbsolutePath());
return file;
public Path getThumbnailsDir(Long userId) throws IOException {
Path thumbnailsDir = getUserDir(userId).resolve(THUMBNAIL_DIR_NAME);
Files.createDirectories(thumbnailsDir);
return thumbnailsDir;
}
public File getUserThumbnailsFile(Long userId, String filename) throws IOException {
if (filename == null || filename.isEmpty()) {
throw new IllegalArgumentException("File name cannot be null or empty");
}
String userDir = getUserFolder(userId).getAbsolutePath();
File file = new File(userDir + File.separator + "thumbnails" + File.separator + filename);
createDirectory(file.getParent());
return file;
public Path getStreamDir(String streamKey) throws IOException {
Path streamFolder = Path.of(streamsDir, streamKey);
Files.createDirectories(streamFolder);
return streamFolder;
}
public void saveMultipartFile(File file, MultipartFile multipartFile) throws IOException {
createDirectory(file.getParent());
Path filePath = Paths.get(file.getAbsolutePath());
Files.copy(multipartFile.getInputStream(), filePath, StandardCopyOption.REPLACE_EXISTING);
public void saveMultipartFile(Path path, MultipartFile multipartFile) throws IOException {
Files.createDirectories(path.getParent());
Files.copy(multipartFile.getInputStream(), path, StandardCopyOption.REPLACE_EXISTING);
}
public void copyFile(File source, File target) throws IOException {
Path sourcePath = Paths.get(source.getAbsolutePath());
Path destPath = Paths.get(target.getAbsolutePath());
Files.createDirectories(destPath.getParent());
Files.copy(sourcePath, destPath, StandardCopyOption.REPLACE_EXISTING);
logger.info("Copied file from {} to {}", sourcePath, destPath);
}
public void cutFile(File source, File target) throws IOException {
copyFile(source, target);
Files.deleteIfExists(source.toPath());
logger.info("Deleted source file: {}", source.getAbsolutePath());
}
private void createDirectory(String dir) throws IOException {
// Create the directory if it doesn't exist
Path outputPath = Paths.get(dir);
if (Files.notExists(outputPath)) {
Files.createDirectories(outputPath);
logger.info("Created directory: {}", outputPath);
}
public void copyFile(Path source, Path target) throws IOException {
Files.createDirectories(target.getParent());
Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING);
logger.info("Copied file from {} to {}", source, target);
}
private void cleanUpDirectory(String dir) throws IOException {
File file = new File(dir);
File[] files = file.listFiles();
Path dirPath = Path.of(dir);
if (files == null) {
if (Files.notExists(dirPath)) {
logger.warn("No files found in directory: {}", dir);
return;
}
for (File f : files){
if (f.isFile() && f.lastModified() < (System.currentTimeMillis() - TEMP_DIR_TIME_LIMIT)) {
Files.delete(f.toPath());
}
try (Stream<Path> files = Files.list(dirPath)) {
files.filter(Files::isRegularFile)
.filter(p -> {
try {
return Files.getLastModifiedTime(p).toMillis()
< System.currentTimeMillis() - TEMP_DIR_TIME_LIMIT;
} catch (IOException e) {
logger.warn("Could not read last modified time for {}", p);
return false;
}
})
.forEach(p -> {
try {
Files.delete(p);
} catch (IOException e) {
logger.warn("Could not delete file {}", p);
}
});
}
}
@PostConstruct
public void createDirectoriesIfNotExist() throws IOException {
createDirectory(tempInputsDir);
createDirectory(tempOutputsDir);
createDirectory(outputDir);
Files.createDirectories(Path.of(tempInputsDir));
Files.createDirectories(Path.of(tempOutputsDir));
Files.createDirectories(Path.of(usersDir));
}
@Scheduled(fixedRate = TEMP_DIR_CLEANUP_RATE)

View File

@@ -10,12 +10,14 @@ import com.ddf.vodsystem.services.media.StreamActionsService;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.time.Instant;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Service
public class MediaService {
@@ -49,8 +51,8 @@ public class MediaService {
Optional<User> user = userService.getLoggedInUser();
String filename = job.getUuid() + ".mp4";
File tempInputFile = directoryService.getTempInputFile(filename);
File tempOutputFile = directoryService.getTempOutputFile(filename);
Path tempInputFile = directoryService.getTempInputDir().resolve(filename);
Path tempOutputFile = directoryService.getTempOutputDir().resolve(filename);
directoryService.saveMultipartFile(tempInputFile, file);
job.setState(JobState.PROCESSING);
@@ -107,8 +109,8 @@ public class MediaService {
.orElseThrow(() -> new NotAuthenticated("User is not authenticated"));
Job job = jobRegistryService.generateJob();
File streamDirectory = directoryService.getStreamDir(user.getStreamKey());
List<File> fileSegments = getSegmentsInRange(streamDirectory.getAbsolutePath(), startTime, endTime);
Path streamDirectory = directoryService.getStreamDir(user.getStreamKey());
List<Path> fileSegments = getSegmentsInRange(streamDirectory, startTime, endTime);
if (fileSegments.isEmpty()) {
throw new IllegalArgumentException("No stream segments found in the given time range");
@@ -119,7 +121,7 @@ public class MediaService {
float duration = (endTime.toEpochMilli() - startTime.toEpochMilli()) / 1000f;
job.setState(JobState.PROCESSING);
File outputFile = directoryService.getVodFile(user.getId(), UUID.randomUUID() + ".mp4");
Path outputFile = directoryService.getVodsDir(user.getId()).resolve(UUID.randomUUID() + ".mp4");
streamActionsService.saveSection(
fileSegments,
trimOffset,
@@ -167,8 +169,8 @@ public class MediaService {
.orElseThrow(() -> new NotAuthenticated("User is not authenticated"));
Job job = jobRegistryService.generateJob();
File streamDirectory = directoryService.getStreamDir(user.getStreamKey());
List<File> fileSegments = getSegmentsInRange(streamDirectory.getAbsolutePath(), startTime, endTime);
Path streamDirectory = directoryService.getStreamDir(user.getStreamKey());
List<Path> fileSegments = getSegmentsInRange(streamDirectory, startTime, endTime);
if (fileSegments.isEmpty()) {
throw new IllegalArgumentException("No stream segments found in the given time range");
@@ -177,7 +179,7 @@ public class MediaService {
long firstSegmentMs = parseTimestampMs(fileSegments.getFirst());
float trimOffset = Math.max(0f, (startTime.toEpochMilli() - firstSegmentMs) / 1000f);
File outputFile = directoryService.getTempOutputFile(UUID.randomUUID() + ".mp4");
Path outputFile = directoryService.getTempOutputDir().resolve(UUID.randomUUID() + ".mp4");
streamActionsService.saveSection(fileSegments, trimOffset, duration, outputFile, job.getProgressTracker())
.thenRun(() -> {
job.setDownload(outputFile);
@@ -188,7 +190,7 @@ public class MediaService {
"",
user,
outputFile,
outputFile.getName()
outputFile.getFileName().toString()
);
}
).exceptionally(ex -> {
@@ -201,25 +203,24 @@ public class MediaService {
return job;
}
private List<File> getSegmentsInRange(String streamDirectory, Instant startTime, Instant endTime) {
File dir = new File(streamDirectory);
File[] tsFiles = dir.listFiles((d, name) -> name.endsWith(".ts"));
if (tsFiles == null) return List.of();
private List<Path> getSegmentsInRange(Path streamDirectory, Instant startTime, Instant endTime) throws IOException {
long startMs = startTime.toEpochMilli();
long endMs = endTime.toEpochMilli();
return Arrays.stream(tsFiles)
.filter(f -> {
long segMs = parseTimestampMs(f);
return segMs < endMs && (segMs + HLS_SEGMENT_LENGTH * 1000) > startMs;
})
.sorted(Comparator.comparingLong(this::parseTimestampMs))
.collect(Collectors.toList());
try (Stream<Path> files = Files.list(streamDirectory)) {
return files
.filter(p -> p.getFileName().toString().endsWith(".ts"))
.filter(p -> {
long segMs = parseTimestampMs(p);
return segMs < endMs && (segMs + HLS_SEGMENT_LENGTH * 1000) > startMs;
})
.sorted(Comparator.comparingLong(this::parseTimestampMs))
.collect(Collectors.toList());
}
}
private long parseTimestampMs(File file) {
String name = file.getName().replace(".ts", "");
private long parseTimestampMs(Path path) {
String name = path.getFileName().toString().replace(".ts", "");
long value = Long.parseLong(name);
// nginx hls_fragment_naming system uses seconds; convert to ms
return value < 1_000_000_000_000L ? value * 1000L : value;

View File

@@ -8,8 +8,8 @@ import org.slf4j.LoggerFactory;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CompletableFuture;
@@ -26,14 +26,18 @@ public class CompressionService {
private final Pattern timePattern = Pattern.compile("out_time_ms=(\\d+)");
@Async("ffmpegTaskExecutor")
public CompletableFuture<CommandOutput> compress(File inputFile,
File outputFile,
public CompletableFuture<CommandOutput> compress(Path inputFile,
Path outputFile,
ClipOptions clipOptions,
ProgressTracker progress
) throws IOException, InterruptedException {
logger.info("Compressing video from {} to {}", inputFile.getAbsolutePath(), outputFile.getAbsolutePath());
logger.info("Compressing video from {} to {}", inputFile.toAbsolutePath(), outputFile.toAbsolutePath());
List<String> command = buildCommand(inputFile, outputFile, clipOptions);
List<String> command = buildCommand(
inputFile,
outputFile,
clipOptions
);
CommandOutput result = CommandRunner.run(command, line -> setProgress(line, progress, clipOptions.getDuration()));
progress.markComplete();
@@ -93,14 +97,14 @@ public class CompressionService {
return command;
}
private List<String> buildInputs(File inputFile, Float startPoint, Float length) {
private List<String> buildInputs(Path inputFile, Float startPoint, Float length) {
List<String> command = new ArrayList<>();
command.add("-ss");
command.add(startPoint.toString());
command.add("-i");
command.add(inputFile.getAbsolutePath());
command.add(inputFile.toAbsolutePath().toString());
command.add("-t");
command.add(Float.toString(length));
@@ -108,7 +112,7 @@ public class CompressionService {
return command;
}
private List<String> buildCommand(File inputFile, File outputFile, ClipOptions clipOptions) {
private List<String> buildCommand(Path inputFile, Path outputFile, ClipOptions clipOptions) {
List<String> command = new ArrayList<>();
command.add("ffmpeg");
command.add("-progress");
@@ -126,7 +130,7 @@ public class CompressionService {
}
// Output file
command.add(outputFile.getAbsolutePath());
command.add(outputFile.toAbsolutePath().toString());
return command;
}
}

View File

@@ -10,8 +10,8 @@ import org.slf4j.LoggerFactory;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Future;
@@ -21,8 +21,8 @@ public class MetadataService {
private static final Logger logger = LoggerFactory.getLogger(MetadataService.class);
@Async("ffmpegTaskExecutor")
public Future<ClipOptions> getVideoMetadata(File file) {
logger.info("Getting metadata for file {}", file.getAbsolutePath());
public Future<ClipOptions> getVideoMetadata(Path file) {
logger.info("Getting metadata for file {}", file);
List<String> command = List.of(
"ffprobe",
@@ -30,7 +30,7 @@ public class MetadataService {
"-print_format", "json",
"-show_format", "-select_streams",
"v:0", "-show_entries", "stream=duration,width,height,r_frame_rate:format=size,duration",
"-i", file.getAbsolutePath()
"-i", file.toAbsolutePath().toString()
);
ObjectMapper mapper = new ObjectMapper();
@@ -51,48 +51,6 @@ public class MetadataService {
}
}
public void validateMetadata(ClipOptions inputFileMetadata, ClipOptions outputFileMetadata) {
Float start = outputFileMetadata.getStartPoint();
Float duration = outputFileMetadata.getDuration();
Float fileSize = outputFileMetadata.getFileSize();
Integer width = outputFileMetadata.getWidth();
Integer height = outputFileMetadata.getHeight();
Float fps = outputFileMetadata.getFps();
if (start == null) {
outputFileMetadata.setStartPoint(0f);
}
if (duration == null) {
outputFileMetadata.setDuration(inputFileMetadata.getDuration());
}
if (start != null && start < 0) {
throw new IllegalArgumentException("Start point cannot be negative");
}
if (duration != null && duration < 0) {
throw new IllegalArgumentException("Duration cannot be negative");
}
if (fileSize != null && fileSize < 100) {
throw new IllegalArgumentException("File size cannot be less than 100kb");
}
if (width != null && width < 1) {
throw new IllegalArgumentException("Width cannot be less than 1");
}
if (height != null && height < 1) {
throw new IllegalArgumentException("Height cannot be less than 1");
}
if (fps != null && fps < 1) {
throw new IllegalArgumentException("FPS cannot be less than 1");
}
}
private ClipOptions parseVideoMetadata(JsonNode node) {
ClipOptions metadata = new ClipOptions();
metadata.setStartPoint(0f);

View File

@@ -6,7 +6,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import java.io.File;
import java.nio.file.Path;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
@@ -47,16 +47,16 @@ public class StreamActionsService {
* failure
*/
@Async("ffmpegTaskExecutor")
public CompletableFuture<File> saveSection(
List<File> segments,
public CompletableFuture<Path> saveSection(
List<Path> segments,
float trimOffset,
float duration,
File outputFile,
Path outputFile,
ProgressTracker progressTracker
) {
try {
String concatInput = segments.stream()
.map(File::getAbsolutePath)
.map(p -> p.toAbsolutePath().toString())
.collect(Collectors.joining("|"));
List<String> command = List.of(
@@ -66,16 +66,16 @@ public class StreamActionsService {
"-ss", String.valueOf(trimOffset),
"-t", String.valueOf(duration),
"-c", "copy",
outputFile.getAbsolutePath()
outputFile.toAbsolutePath().toString()
);
logger.info("Saving section with ({} segments) to '{}'",
segments.size(), outputFile.getAbsolutePath());
segments.size(), outputFile);
CommandRunner.run(command, line -> CommandRunner.setProgress(line, progressTracker, duration));
return CompletableFuture.completedFuture(outputFile);
} catch (Exception e) {
logger.error("Failed to save section to '{}'", outputFile.getAbsolutePath(), e);
logger.error("Failed to save section to '{}'", outputFile, e);
return CompletableFuture.failedFuture(e);
}
}

View File

@@ -5,8 +5,8 @@ import org.slf4j.LoggerFactory;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
@Service
@@ -14,15 +14,15 @@ public class ThumbnailService {
private static final Logger logger = LoggerFactory.getLogger(ThumbnailService.class);
@Async("ffmpegTaskExecutor")
public void createThumbnail(File inputFile, File outputFile, Float timeInVideo) throws IOException, InterruptedException {
public void createThumbnail(Path inputFile, Path outputFile, Float timeInVideo) throws IOException, InterruptedException {
logger.info("Creating thumbnail at {} seconds", timeInVideo);
List<String> command = List.of(
"ffmpeg",
"-ss", timeInVideo.toString(),
"-i", inputFile.getAbsolutePath(),
"-i", inputFile.toAbsolutePath().toString(),
"-frames:v", "1",
outputFile.getAbsolutePath()
outputFile.toAbsolutePath().toString()
);
CommandRunner.run(command);

View File

@@ -1 +1,6 @@
spring.docker.compose.file=docker-compose.dev.yml
spring.docker.compose.file=docker-compose.dev.yml
media.temp.inputs=./video/inputs
media.temp.outputs=./videos/outputs
media.users=${MEDIA_DATA}
media.streams=${STREAM_DATA}

View File

@@ -3,9 +3,6 @@ spring.application.name=vodSystem
# VODs
spring.servlet.multipart.max-file-size=2GB
spring.servlet.multipart.max-request-size=2GB
storage.temp.inputs=videos/inputs/
storage.temp.outputs=videos/outputs/
storage.outputs=${MEDIA_DATA}
# Database
spring.datasource.url=${POSTGRES_URL}
@@ -21,15 +18,17 @@ spring.sql.init.schema-locations=classpath:db/schema.sql
spring.security.oauth2.client.registration.google.client-id=${GOOGLE_CLIENT_ID}
spring.security.oauth2.client.registration.google.client-secret=${GOOGLE_CLIENT_SECRET}
spring.security.oauth2.client.registration.google.scope=profile,email
jwt.secret.key=${JWT_SECRET_KEY}
jwt.expiration=3600000
# Frontend
frontend.url=${FRONTEND_URL}
# Stream HLS
storage.stream=${STREAM_DATA}
# Paths
media.temp.inputs=/tmp/inputs/
media.temp.outputs=/tmp/outputs/
media.users=/app/media/
media.streams=/app/stream/
## Server Configuration
server.servlet.session.timeout=30m