Merge pull request 'playlist_service' (#2) from playlist_service into main

Reviewed-on: #2

.gitignore

@@ -2,6 +2,7 @@ HELP.md
 target/
 .mvn/wrapper/maven-wrapper.jar
 !**/src/main/**/target/
+downloads/
 !**/src/test/**/target/
 
 ### STS ###

pom.xml

@@ -44,14 +44,22 @@
 			<groupId>org.springframework.boot</groupId>
 			<artifactId>spring-boot-starter-webmvc</artifactId>
 		</dependency>
-
 		<dependency>
 			<groupId>org.projectlombok</groupId>
 			<artifactId>lombok</artifactId>
 			<version>1.18.40</version>
 			<scope>provided</scope>
 		</dependency>
-
+		<dependency>
+			<groupId>org.flywaydb</groupId>
+			<artifactId>flyway-database-postgresql</artifactId>
+			<version>12.0.3</version>
+			<scope>runtime</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework.boot</groupId>
+			<artifactId>spring-boot-starter-flyway</artifactId>
+		</dependency>
 		<dependency>
 			<groupId>org.postgresql</groupId>
 			<artifactId>postgresql</artifactId>
@@ -67,6 +75,12 @@
 			<artifactId>spring-boot-starter-webmvc-test</artifactId>
 			<scope>test</scope>
 		</dependency>
+		<dependency>
+			<groupId>org.jetbrains</groupId>
+			<artifactId>annotations</artifactId>
+			<version>26.1.0</version>
+			<scope>compile</scope>
+		</dependency>
 	</dependencies>
 
 	<build>
@@ -75,6 +89,17 @@
 				<groupId>org.springframework.boot</groupId>
 				<artifactId>spring-boot-maven-plugin</artifactId>
 			</plugin>
+			<plugin>
+			<groupId>org.flywaydb</groupId>
+			<artifactId>flyway-maven-plugin</artifactId>
+			<version>12.0.1</version>
+				<configuration>
+					<url>jdbc:postgresql://127.0.0.1:5432/videos</url>
+					<user>postgres</user>
+					<password>5995</password>
+					<cleanDisabled>false</cleanDisabled>
+				</configuration>
+		</plugin>
 		</plugins>
 	</build>
 

src/main/java/com/example/video_downloader/VideoDownloaderApplication.java

@@ -10,4 +10,4 @@ public class VideoDownloaderApplication {
 		SpringApplication.run(VideoDownloaderApplication.class, args);
 	}
 
-}
+}

src/main/java/com/example/video_downloader/controllers/VideoController.java

@@ -1,39 +1,49 @@
 package com.example.video_downloader.controllers;
 
+import com.example.video_downloader.dto.SaveNewPlaylistRequest;
 import com.example.video_downloader.dto.SaveNewVideoRequest;
+import com.example.video_downloader.entity.Playlist;
 import com.example.video_downloader.entity.Video;
+import com.example.video_downloader.services.PlaylistService;
 import com.example.video_downloader.services.VideoService;
+import com.example.video_downloader.services.YTDLPService;
 import lombok.Data;
+import org.jetbrains.annotations.NotNull;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.*;
-import org.springframework.web.server.ResponseStatusException;
 
+import java.io.IOException;
 import java.util.List;
-import java.util.Optional;
 
 @RestController
-@RequestMapping(path="/video")
+@RequestMapping(path="/v2")
 @CrossOrigin(origins = "http://localhost:3000")
 public class VideoController {
 
     @Autowired
     private VideoService videoService;
 
+    @Autowired
+    private PlaylistService playlistService;
+
+    @Autowired
+    private YTDLPService ytdlpService;
+
+
     @Data
-    private class Response {
-        private Long id;
+    private static class Trigger {
+        private boolean input;
     }
 
     @PostMapping(path = "/videos")
-    public Response saveNewVideo(@RequestBody SaveNewVideoRequest request){
+    public Video saveNewVideo(@RequestBody SaveNewVideoRequest request){
+        return videoService.saveNewVideo(request);
+    }
 
-        Response response = new Response();
-        Video video = videoService.saveNewVideo(request);
-        response.setId(video.getId());
-
-        return response;
+    @PostMapping(path = "/playlists")
+    public void saveNewPlaylist(@RequestBody SaveNewPlaylistRequest request){
+        Playlist playlist = playlistService.saveNewPlaylist(request);
 
     }
 
@@ -47,4 +57,17 @@ public class VideoController {
         return ResponseEntity.ok(videoService.getVideoById(id));
 
     }
+
+    @PostMapping(path = "/fire")
+    public int getJobDone(@RequestBody @NotNull Trigger trigger) throws IOException, InterruptedException {
+
+        int statusCode = 200;
+        if(trigger.input) {
+            ytdlpService.processPlaylist();
+            ytdlpService.validateVideos();
+            ytdlpService.downloadVideos();
+        }
+        return statusCode;
+
+    }
 }

src/main/java/com/example/video_downloader/dto/SaveNewPlaylistRequest.java

@@ -0,0 +1,9 @@
+package com.example.video_downloader.dto;
+
+import lombok.Data;
+
+@Data
+public class SaveNewPlaylistRequest {
+    private String playlistName;
+    private String url;
+}

src/main/java/com/example/video_downloader/entity/Creator.java

@@ -0,0 +1,20 @@
+package com.example.video_downloader.entity;
+
+import jakarta.persistence.*;
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+@Entity
+public class Creator {
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    private Long id;
+
+    private String name;
+    private String url;
+
+    @OneToMany(mappedBy = "creator")
+    private List<Video> videos;
+}

src/main/java/com/example/video_downloader/entity/Playlist.java

@@ -0,0 +1,36 @@
+package com.example.video_downloader.entity;
+
+import jakarta.persistence.*;
+import lombok.Data;
+import lombok.ToString;
+
+import java.util.List;
+
+@Data
+@ToString
+@Entity
+@Table(name = "playlists")
+
+public class Playlist {
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    private Long id;
+
+    @Column(name = "playlist_name", nullable = false)
+    private String playlistName;
+    
+    @Column
+    private String url;
+
+    @Enumerated(EnumType.STRING)
+    private StatusEnum status;
+
+    @ManyToMany
+    @JoinTable(
+            name = "video_playlist",
+            joinColumns = @JoinColumn(name = "playlist_id"),
+            inverseJoinColumns = @JoinColumn(name = "video_id")
+    )
+    private List<Video> videos;
+
+}

src/main/java/com/example/video_downloader/entity/Video.java

@@ -4,21 +4,43 @@ import jakarta.persistence.*;
 import lombok.Data;
 import lombok.ToString;
 
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
 @Data
 @ToString
 @Entity
 @Table(name = "videos")
 public class Video {
     @Id
-    @GeneratedValue(strategy = GenerationType.SEQUENCE)
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
     private Long id;
 
     private String url;
-    private String ytdl;
-    private String ytMetaData;
+    private String name;
+
+    @Column(name = "upload_date")
+    private String uploadDate;
+
+    private String duration;
 
     @Enumerated(EnumType.STRING)
     private StatusEnum status;
 
+    @Column(name="full_path")
     private String fullPath;
+
+    @ManyToMany
+    @JoinTable(
+            name = "video_playlist",
+            joinColumns = @JoinColumn(name = "video_id"),
+            inverseJoinColumns = @JoinColumn(name = "playlist_id")
+    )
+    private List<Playlist> playlist = new ArrayList<>();
+
+    @OneToOne
+    @JoinColumn(name = "creator_id")
+    private Creator creator;
+
 }

src/main/java/com/example/video_downloader/repositories/CreatorRepository.java

@@ -0,0 +1,10 @@
+package com.example.video_downloader.repositories;
+
+import org.springframework.data.jpa.repository.JpaRepository;
+import com.example.video_downloader.entity.Creator;
+
+import java.util.Optional;
+
+public interface CreatorRepository extends JpaRepository<Creator, Long> {
+    Optional<Creator> findByName(String name);
+}

src/main/java/com/example/video_downloader/repositories/PlaylistRepository.java

@@ -0,0 +1,11 @@
+package com.example.video_downloader.repositories;
+
+import com.example.video_downloader.entity.Playlist;
+import com.example.video_downloader.entity.StatusEnum;
+import org.springframework.data.jpa.repository.JpaRepository;
+
+import java.util.List;
+
+public interface PlaylistRepository extends JpaRepository<Playlist, Long> {
+    List<Playlist> findByStatus(StatusEnum status);
+}

src/main/java/com/example/video_downloader/repositories/VideoRepository.java

@@ -1,9 +1,13 @@
 package com.example.video_downloader.repositories;
 
+import com.example.video_downloader.entity.StatusEnum;
 import com.example.video_downloader.entity.Video;
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.stereotype.Repository;
+import java.util.ArrayList;
+import java.util.List;
 
 @Repository
 public interface VideoRepository extends JpaRepository<Video, Long> {
+    List<Video> findByStatus(StatusEnum status);
 }

src/main/java/com/example/video_downloader/services/PlaylistService.java

@@ -0,0 +1,32 @@
+package com.example.video_downloader.services;
+
+import com.example.video_downloader.dto.SaveNewPlaylistRequest;
+import com.example.video_downloader.entity.Playlist;
+import com.example.video_downloader.entity.StatusEnum;
+import com.example.video_downloader.repositories.PlaylistRepository;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.security.InvalidParameterException;
+import java.util.List;
+
+@Service
+public class PlaylistService {
+
+    @Autowired
+    private PlaylistRepository playlistRepository;
+
+    public Playlist saveNewPlaylist(SaveNewPlaylistRequest request){
+        if(!request.getUrl().contains("list")) throw new InvalidParameterException();
+        Playlist playlist = new Playlist();
+        playlist.setStatus(StatusEnum.NEW);
+        playlist.setPlaylistName(request.getPlaylistName());
+        playlist.setUrl(request.getUrl());
+        playlistRepository.save(playlist);
+        return playlist;
+    }
+
+    public List<Playlist> getAllPlaylists(){
+        return playlistRepository.findAll();
+    }
+}

src/main/java/com/example/video_downloader/services/ProcessService.java

@@ -1,25 +1,21 @@
 package com.example.video_downloader.services;
 
-import jakarta.annotation.PostConstruct;
 import org.springframework.stereotype.Service;
-
 import java.io.IOException;
-import java.nio.charset.StandardCharsets;
+
 
 @Service
 public class ProcessService {
 
-    @PostConstruct
-    public void init() throws InterruptedException, IOException {
-        execute("yt-dlp", "");
-    }
-
-    public void execute(String... command) throws InterruptedException, IOException {
+    public String execute(String... command) throws InterruptedException, IOException {
         ProcessBuilder builder = new ProcessBuilder(command);
-        builder.redirectOutput(ProcessBuilder.Redirect.INHERIT);
-        builder.redirectError(ProcessBuilder.Redirect.INHERIT);
+//        builder.redirectOutput(ProcessBuilder.Redirect.INHERIT);
+//        builder.redirectError(ProcessBuilder.Redirect.INHERIT);
         Process process = builder.start();
-        process.waitFor();
+        String res = new String(process.getInputStream().readAllBytes());
+
+        int exitCode = process.waitFor();
+        return res;
     }
 
 

src/main/java/com/example/video_downloader/services/VideoService.java

@@ -4,11 +4,13 @@ import com.example.video_downloader.dto.SaveNewVideoRequest;
 import com.example.video_downloader.entity.StatusEnum;
 import com.example.video_downloader.entity.Video;
 import com.example.video_downloader.repositories.VideoRepository;
+import jakarta.persistence.EntityNotFoundException;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
+import java.security.InvalidParameterException;
+import java.util.ArrayList;
 import java.util.List;
-import java.util.Optional;
 
 @Service
 public class VideoService {
@@ -16,7 +18,8 @@ public class VideoService {
     @Autowired
     private VideoRepository videoRepository;
 
-    public Video saveNewVideo(SaveNewVideoRequest request){
+    public Video saveNewVideo(SaveNewVideoRequest request) {
+        if (request.getUrl().contains("list")) throw new InvalidParameterException("");
         Video video  = new Video();
         video.setStatus(StatusEnum.NEW);
         video.setUrl(request.getUrl());
@@ -30,6 +33,7 @@ public class VideoService {
     }
 
     public Video getVideoById(Long id){
+        //FIXME Throwolj EntityNotFoundException-t
         return videoRepository.findById(id)
                 .orElseThrow(() -> new RuntimeException("Video not found with id: " + id));
     }

src/main/java/com/example/video_downloader/services/YTDLPService.java

@@ -0,0 +1,129 @@
+package com.example.video_downloader.services;
+
+import com.example.video_downloader.entity.Creator;
+import com.example.video_downloader.entity.Playlist;
+import com.example.video_downloader.entity.StatusEnum;
+import com.example.video_downloader.entity.Video;
+import com.example.video_downloader.repositories.CreatorRepository;
+import com.example.video_downloader.repositories.PlaylistRepository;
+import com.example.video_downloader.repositories.VideoRepository;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import org.springframework.web.client.HttpClientErrorException;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+
+@Service
+public class YTDLPService {
+
+    Logger LOGGER = LoggerFactory.getLogger(YTDLPService.class);
+    /* FIXME
+     *  LOGGER.debug("Valami szoveg: {}", valtozo);
+     *  LOGGER.warn("Valami szoveg: {}", valtozo);
+     *  LOGGER.info("Valami szoveg: {}", valtozo);
+     *  LOGGER.error("Valami szoveg: {}", valtozo);
+     *  mindegyik ugyan ugy mukodik, csak a log levelt be lehet allitani propertykent es akkor ha mondjuk error a loglevel akkor a debug logokat nem irja ki csak ami errorkent van logolva
+     *
+     * */
+    @Autowired
+    private VideoRepository videoRepository;
+
+    @Autowired
+    private PlaylistService playlistService;
+
+    @Autowired
+    private PlaylistRepository playlistRepository;
+
+    @Autowired
+    private ProcessService processService;
+
+    @Autowired
+    private CreatorRepository creatorRepository;
+
+
+    public void processPlaylist() throws IOException, InterruptedException {
+        List<Playlist> playlists = playlistRepository.findByStatus(StatusEnum.NEW);
+        for(Playlist playlist : playlists) {
+            String urls =  processService.execute("yt-dlp", "--force-ipv4", "--flat-playlist", "--print", "webpage_url", playlist.getUrl());
+            for(String url : urls.split("\\n")){
+                Video video = new Video();
+                video.setUrl(url);
+                video.setStatus(StatusEnum.NEW);
+                video.getPlaylist().add(playlist);
+                videoRepository.save(video);
+            }
+            /* FIXME itt ugye fogod a playlistet es lemented az osszes hozza tartozo videot.
+               Ha ezzel vegeztel, akkor a playlist-nek a statuszat at kene allitani NEW-rol, *valamire*
+               Kene keresni valami elnevezest arra hogy "a playlistben levo videokat meg nem toltottem le, de az video entityket mar letrehoztam"
+               Pl.: PROCESSED bar nem vagyok benne biztos hogy ez lenne a legjobb nev ra.
+               Amit mondani akarok az az hogy ha ez a methodus lefutott, akkor ha ranezek a DB-re akkor nem tudom megkulonboztetni egy playlisten hogy csak most lett bekuldve,
+               vs mar elkezdtuk feldolgozni, es lementeni a hozza tartozo videokat.
+             * */
+
+        }
+
+    }
+
+    public void validateVideos() throws HttpClientErrorException, IOException, InterruptedException {
+        List<Video> response = videoRepository.findByStatus(StatusEnum.NEW);
+
+        for(Video video : response) {
+                String data = processVideo(video.getUrl());
+                String[] parts = data.trim().split(", ");
+                    video.setName(parts[0]);
+                    video.setUploadDate(parts[2]);
+                    video.setDuration(parts[3]);
+
+                    String creatorName = parts[1];
+                    String creatorUrl = parts[4];
+
+                    Creator creator = creatorRepository
+                            .findByName(creatorName)
+                            .orElseGet(() -> {
+                                Creator newCreator = new Creator();
+                                newCreator.setName(creatorName);
+                                newCreator.setUrl(creatorUrl);
+                                return creatorRepository.save(newCreator);
+                            });
+                    video.setCreator(creator);
+                videoRepository.save(video);
+        }
+    }
+
+    public void downloadVideos() throws IOException, InterruptedException {
+        List<Video> response = videoRepository.findByStatus(StatusEnum.NEW);
+
+        for(Video video : response){
+            downloadVideo(video.getUrl(), video.getId());
+            Path filePath = Paths.get("downloads/" + video.getId() + ".webm");
+            if(Files.exists(filePath) && Files.isRegularFile(filePath)) {
+                System.out.println("Apparently it exists mate.");
+                video.setFullPath(String.valueOf(filePath));
+                video.setStatus(StatusEnum.SAVED);
+            } else {
+                System.out.println("Download failed.");
+                video.setStatus(StatusEnum.FAILED);
+            }
+            videoRepository.save(video);
+        }
+
+    }
+
+
+    public String processVideo(String url) throws HttpClientErrorException, InterruptedException, IOException {
+        return processService.execute("yt-dlp", "--force-ipv4", "--skip-download", "--print", "%(title)s, %(uploader)s, %(upload_date)s, %(duration_string)s, %(uploader_url)s", url);
+    }
+    public void downloadVideo(String url, Long id) throws HttpClientErrorException, InterruptedException, IOException {
+        String outputPath = "downloads/" + id + ".%(ext)s";
+        String result = processService.execute("yt-dlp", "--force-ipv4", "-o", outputPath, url);
+        System.out.println(result);
+    }
+
+
+}

src/main/resources/application.properties

@@ -1,5 +1,9 @@
 spring.application.name=video_downloader
-spring.datasource.url = jdbc:postgresql://localhost:5432/videos
+spring.datasource.url = jdbc:postgresql://127.0.0.1:5432/videos
 spring.datasource.username = postgres
 spring.datasource.password = 5995
-spring.jpa.hibernate.ddl-auto = update
+spring.jpa.hibernate.ddl-auto = none
+logging.level.org.flywaydb=DEBUG
+logging.level.org.springframework.boot.autoconfigure.flyway=DEBUG
+spring.flyway.enabled=true
+spring.flyway.baseline-on-migrate=true

src/main/resources/db/migration/V1__create_table_playlist.sql

@@ -0,0 +1,6 @@
+CREATE TABLE playlists (
+id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+playlist_name VARCHAR(100) NOT NULL,
+url VARCHAR(200) NOT NULL,
+status VARCHAR(10) NOT NULL
+);

src/main/resources/db/migration/V2__create_table_videos.sql

@@ -0,0 +1,9 @@
+CREATE TABLE videos (
+    id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+    name VARCHAR(200),
+    url VARCHAR(200) NOT NULL,
+    upload_date VARCHAR(100),
+    duration VARCHAR(10),
+    status VARCHAR(10) NOT NULL,
+    full_path VARCHAR(100)
+);

src/main/resources/db/migration/V3__create_table_creator.sql

@@ -0,0 +1,5 @@
+CREATE TABLE creator (
+    id BIGSERIAL PRIMARY KEY,
+    name VARCHAR(200) NOT NULL,
+    url VARCHAR(200) NOT NULL
+);

src/main/resources/db/migration/V4__create_table_video_playlist.sql

@@ -0,0 +1,7 @@
+CREATE TABLE video_playlist (
+    video_id BIGINT NOT NULL,
+    playlist_id BIGINT NOT NULL,
+    PRIMARY KEY (video_id, playlist_id),
+    FOREIGN KEY (video_id) REFERENCES videos(id) ON DELETE CASCADE,
+    FOREIGN KEY (playlist_id) REFERENCES playlists(id) ON DELETE CASCADE
+);

src/main/resources/db/migration/V5__alter_table_videos.sql

@@ -0,0 +1,4 @@
+ALTER TABLE videos
+ADD COLUMN creator_id BIGINT,
+ADD CONSTRAINT fk_videos_creator FOREIGN KEY (creator_id) REFERENCES creator(id)
+