sqlite refactor

.dockerignore

@@ -0,0 +1,2 @@
+db/
+yt-downloader/

Dockerfile

@@ -1,11 +1,15 @@
-FROM alpine:3.22.4
+FROM alpine:3.22.4 as base
 USER root
 RUN apk upgrade --no-cache
-RUN apk add --no-cache ffmpeg mutagen curl
+RUN apk add --no-cache ffmpeg mutagen curl sqlite deno bash
 RUN curl -L https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /yt-dlp
 RUN chmod a+rx /yt-dlp
-COPY . .
+
+FROM base as downloader
+COPY download.sh /download.sh
 RUN mkdir /download
 RUN mkdir /music
-RUN chmod +x download.sh
+RUN touch /music/archive.dat
-CMD ["sh", "./download.sh"]
+RUN mkdir /db
+RUN chmod +x /download.sh
+CMD ["bash", "/download.sh"]

docker-compose.yaml

@@ -4,13 +4,16 @@ services:
     image: yt-downloader
     container_name: yt-downloader
     volumes:
-      - ./yt-downloader/:/music
+      - ./music/:/music
+      - ./db/:/db
+    #  - ./download.sh:/download.sh
     restart: "unless-stopped"
     environment:
-      PLAYLIST_URL: "{}"
+      PLAYLIST_URL: "https://www.youtube.com/playlist?list=PL4QRSQmrCreF3dSp3RokdWmS50o0QNesJ"
       FILE_NAME_FORMAT: "%(title)s;%(uploader)s"
-    #deploy:
+      DB: "/db/yt.db"
-      #resources:
+    deploy:
-        #limits:
+      resources:
-          #cpus: '0.005'
+        limits:
-          #memory: 1000M
+          cpus: '0.005'
+          memory: 325M

download.sh

@@ -1,6 +1,57 @@
 #!/bin/bash
-apk update && apk upgrade
+#./yt-dlp -U
-cp /music/archive.dat /download
+
-./yt-dlp -U
+TABLE="downloads"
-./yt-dlp -P "/download" --download-archive "/download/archive.dat" --sleep-interval 5 -x -o ${FILE_NAME_FORMAT} --add-metadata --embed-thumbnail --embed-chapters ${PLAYLIST_URL} 
+CACHE="cache"
-cp /download/* /music
+S_NEW=0
+S_DOWNLOADED=1
+S_MISSING_FROM_PLAYLIST=2
+S_DELETED=3
+S_FAILED=4
+
+#CREATE database
+if [ ! -f "$DB" ]; then
+    echo "Database not found. Creating database"
+    sqlite3 "$DB" "CREATE TABLE $TABLE (id INTEGER PRIMARY KEY,yt_id varchar(20) UNIQUE, file_name TEXT, status INTEGER);"
+    sqlite3 "$DB" "CREATE TABLE $CACHE (id INTEGER PRIMARY KEY,yt_id varchar(20) UNIQUE);"
+fi
+
+#Clear Cache
+sqlite3 "$DB" "DELETE FROM $CACHE ;"
+
+#Fill Cache
+./yt-dlp --get-id --flat-playlist "${PLAYLIST_URL}" | while read -r ID; do
+    #echo "Inserting video ID: $ID"
+    sqlite3 "$DB" "INSERT INTO $CACHE (yt_id) VALUES ('$ID');" 2>/dev/null
+done
+
+#Download music existing in cache but not in downloads table
+sqlite3 -list -noheader "$DB" "SELECT yt_id FROM $CACHE" | while IFS='|' read -r ID; do
+    count=$(sqlite3 "$DB" "SELECT count(*) FROM $TABLE WHERE yt_id = '$ID' AND status = $S_DOWNLOADED")
+
+    if [ $count -eq 0 ];then
+        ./yt-dlp -P "/download" --sleep-interval 5 -x -o ${FILE_NAME_FORMAT} --add-metadata --embed-thumbnail --embed-chapters $ID
+        name=$(./yt-dlp --get-filename -o ${FILE_NAME_FORMAT} $ID)
+
+        if [ $? -eq 0 ]; then
+            sqlite3 "$DB" "INSERT INTO $TABLE (yt_id,status,file_name) VALUES ('$ID', $S_DOWNLOADED,${name@Q});"
+            mv /download/* /music/
+        else
+            sqlite3 "$DB" "INSERT INTO $TABLE (yt_id,status,file_name) VALUES ('$ID', $S_FAILED,${name@Q});"
+        fi
+    fi
+done
+
+#Delete files existing in downloads but not in cache
+sqlite3 -list -noheader "$DB" "SELECT $TABLE.file_name FROM $TABLE LEFT OUTER JOIN $CACHE ON $TABLE.yt_id = $CACHE.yt_id WHERE $CACHE.ID IS NULL AND $TABLE.status != '3' " | while IFS='|' read -r file_name; do
+
+    echo "DELETING FILE WITH NAME: ${file_name@Q}"
+    sqlite3 "$DB" "UPDATE $TABLE SET status = '$S_DELETED' where file_name=${file_name@Q};"
+    echo "/music/${file_name@Q}*"
+    rm "/music/${file_name@Q}"*
+done
+#cp /music/archive.dat /download
+#./yt-dlp -U
+#./yt-dlp -P "/download" --download-archive "/download/archive.dat" --sleep-interval 5 -x -o ${FILE_NAME_FORMAT} --add-metadata --embed-thumbnail --embed-chapters ${PLAYLIST_URL} 
+#cp /download/* /music
+