diff --git a/fetch-playlist.sh b/fetch-playlist.sh index d71dbed..7b3adcb 100755 --- a/fetch-playlist.sh +++ b/fetch-playlist.sh @@ -6,162 +6,84 @@ # Configuration PUBKEY="3f11abb2e235da2d4dda5d6deb2f123173476a745a3ca56895a1d0f632a42f40" -RELAY="wss://drops.basspistol.org" +RELAY="wss://tortellino.basspistol.org" DATA_DIR="data/playlists" -NAK_BIN="nak" -JQ_BIN="jq" +RAW_DIR="data/playlists/raw" -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' +mkdir -p "$DATA_DIR" "$RAW_DIR" -mkdir -p "$DATA_DIR" - -log() { echo -e "${BLUE}[$(date '+%Y-%m-%d %H:%M:%S')]${NC} $1"; } -log_success() { echo -e "${GREEN}✅ $1${NC}"; } -log_warning() { echo -e "${YELLOW}⚠️ $1${NC}"; } -log_error() { echo -e "${RED}❌ $1${NC}"; } +log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1"; } # ============================================ -# Step 1: Fetch playlist events +# Step 1: Fetch playlists and tracks # ============================================ log "Fetching playlist events (kind 34139) from $RELAY..." - -PLAYLISTS_JSON=$(mktemp) -$NAK_BIN req -k 34139 -a "$PUBKEY" "$RELAY" | $JQ_BIN -s '.' > "$PLAYLISTS_JSON" - -if [ ! -s "$PLAYLISTS_JSON" ]; then - log_error "No playlist data received." - rm -f "$PLAYLISTS_JSON" - exit 1 -fi - -PLAYLIST_COUNT=$(cat "$PLAYLISTS_JSON" | $JQ_BIN 'length') -log_success "Found $PLAYLIST_COUNT playlist(s)" - -# ============================================ -# Step 2: Fetch ALL tracks -# ============================================ +nak req -k 34139 -a "$PUBKEY" "$RELAY" > "$RAW_DIR/playlists-raw.jsonl" log "Fetching ALL tracks (kind 36787) from $PUBKEY..." - -ALL_TRACKS_JSON=$(mktemp) -$NAK_BIN req -k 36787 -a "$PUBKEY" "$RELAY" | $JQ_BIN -s '.' > "$ALL_TRACKS_JSON" - -TRACK_COUNT=$(cat "$ALL_TRACKS_JSON" | $JQ_BIN 'length' 2>/dev/null || echo "0") -log_success "Found $TRACK_COUNT total tracks from pubkey" +nak req -k 36787 -a "$PUBKEY" "$RELAY" > "$RAW_DIR/tracks-raw.jsonl" # ============================================ -# Step 3: Build playlist data +# Step 2: Convert JSONL to JSON arrays +# ============================================ + +log "Converting raw data to JSON arrays..." + +# Convert jsonl to json arrays +cat "$RAW_DIR/playlists-raw.jsonl" | jq -s '.' > "$RAW_DIR/playlists-raw.json" +cat "$RAW_DIR/tracks-raw.jsonl" | jq -s '.' > "$RAW_DIR/tracks-raw.json" + +# ============================================ +# Step 3: Build playlist data with jq # ============================================ log "Building playlist data structure..." -OUTPUT_FILE="$DATA_DIR/playlists.json" -FLATTENED_OUTPUT="$DATA_DIR/tracks.json" - -# Create a file with track lookup by d-tag -TRACK_LOOKUP=$(mktemp) -cat "$ALL_TRACKS_JSON" | $JQ_BIN -r ' - .[] | - . as $track | - (.tags[] | select(.[0] == "d") | .[1]) as $d_tag | - "\($d_tag)|\($track | tojson)" -' > "$TRACK_LOOKUP" - -# Build the playlists JSON -echo '{"playlists": []}' > "$OUTPUT_FILE" - -# Process each playlist -cat "$PLAYLISTS_JSON" | $JQ_BIN -c '.[]' | while read -r playlist; do - # Extract playlist metadata - PLAYLIST_ID=$(echo "$playlist" | $JQ_BIN -r '.id') - PLAYLIST_CREATED=$(echo "$playlist" | $JQ_BIN -r '.created_at') - - # FIX: Extract name from "title" tag (not "name") - PLAYLIST_NAME=$(echo "$playlist" | $JQ_BIN -r '.tags[] | select(.[0] == "title") | .[1] // ""') - PLAYLIST_DESC=$(echo "$playlist" | $JQ_BIN -r '.tags[] | select(.[0] == "description") | .[1] // ""') - PLAYLIST_IMAGE=$(echo "$playlist" | $JQ_BIN -r '.tags[] | select(.[0] == "image") | .[1] // ""') - - # Debug: show what was extracted - echo " Playlist: '$PLAYLIST_NAME'" >&2 - - # Get track references from 'a' tags - TRACK_REFS=$(echo "$playlist" | $JQ_BIN -r '.tags[] | select(.[0] == "a") | .[1]') - - # Build tracks array for this playlist - TRACKS_JSON="[]" - - while read -r ref; do - if [ -n "$ref" ]; then - # Extract identifier (the part after the second colon) - IDENTIFIER=$(echo "$ref" | cut -d':' -f3-) - - # Find the track in the lookup - TRACK_DATA=$(grep "^$IDENTIFIER|" "$TRACK_LOOKUP" | head -1 | cut -d'|' -f2-) - - if [ -n "$TRACK_DATA" ] && [ "$TRACK_DATA" != "null" ]; then - # Extract track fields - TRACK_TITLE=$(echo "$TRACK_DATA" | $JQ_BIN -r '.tags[] | select(.[0] == "title") | .[1] // "Untitled"') - TRACK_ARTIST=$(echo "$TRACK_DATA" | $JQ_BIN -r '.tags[] | select(.[0] == "artist") | .[1] // "Unknown Artist"') - TRACK_ALBUM=$(echo "$TRACK_DATA" | $JQ_BIN -r '.tags[] | select(.[0] == "album") | .[1] // ""') - TRACK_URL=$(echo "$TRACK_DATA" | $JQ_BIN -r '.tags[] | select(.[0] == "url") | .[1] // ""') - TRACK_IMAGE=$(echo "$TRACK_DATA" | $JQ_BIN -r '.tags[] | select(.[0] == "image") | .[1] // ""') - TRACK_DURATION=$(echo "$TRACK_DATA" | $JQ_BIN -r '.tags[] | select(.[0] == "duration") | .[1] // ""') - TRACK_ID=$(echo "$TRACK_DATA" | $JQ_BIN -r '.id') - TRACK_CREATED=$(echo "$TRACK_DATA" | $JQ_BIN -r '.created_at') - TRACK_TAGS=$(echo "$TRACK_DATA" | $JQ_BIN -r '[.tags[] | select(.[0] == "t") | .[1]] | join(",")') - - # Build track JSON object - TRACK_JSON=$(cat < "${OUTPUT_FILE}.tmp" - mv "${OUTPUT_FILE}.tmp" "$OUTPUT_FILE" -done +# Build the playlists.json using the json files +jq -n \ + --slurpfile playlists "$RAW_DIR/playlists-raw.json" \ + --slurpfile tracks "$RAW_DIR/tracks-raw.json" \ + '{ + playlists: [ + $playlists[] | .[] | + select(.kind == 34139) | + { + id: .id, + created_at: .created_at, + name: (.tags[] | select(.[0] == "title") | .[1] // ""), + description: (.tags[] | select(.[0] == "description") | .[1] // ""), + image: (.tags[] | select(.[0] == "image") | .[1] // ""), + tracks: [ + .tags[] | + select(.[0] == "a") | .[1] as $ref | + $tracks[] | .[] | + select( + ($ref | split(":")[2]) == (.tags[] | select(.[0] == "d") | .[1]) + ) | + { + id: .id, + created_at: .created_at, + title: (.tags[] | select(.[0] == "title") | .[1] // "Untitled"), + artist: (.tags[] | select(.[0] == "artist") | .[1] // "Unknown Artist"), + album: (.tags[] | select(.[0] == "album") | .[1] // ""), + url: (.tags[] | select(.[0] == "url") | .[1] // ""), + image: (.tags[] | select(.[0] == "image") | .[1] // ""), + duration: (.tags[] | select(.[0] == "duration") | .[1] // ""), + tags: [.tags[] | select(.[0] == "t") | .[1]] + } + ] + } + ] + }' > "$DATA_DIR/playlists.json" # ============================================ -# Step 4: Flattened tracks file +# Step 4: Generate flattened tracks file # ============================================ log "Generating flattened tracks file..." -cat "$ALL_TRACKS_JSON" | $JQ_BIN ' +cat "$RAW_DIR/tracks-raw.json" | jq ' .[] | { id: .id, created_at: .created_at, @@ -174,41 +96,34 @@ cat "$ALL_TRACKS_JSON" | $JQ_BIN ' tags: [.tags[] | select(.[0] == "t") | .[1]], d_tag: (.tags[] | select(.[0] == "d") | .[1] // "") } -' | $JQ_BIN -s '.' > "$FLATTENED_OUTPUT" +' | jq -s '.' > "$DATA_DIR/tracks.json" # ============================================ -# Step 5: Save raw data +# Step 5: Cleanup raw jsonl files # ============================================ -cp "$PLAYLISTS_JSON" "$DATA_DIR/playlists-raw.json" -cp "$ALL_TRACKS_JSON" "$DATA_DIR/tracks-raw.json" +rm -f "$RAW_DIR"/*.jsonl # ============================================ -# Step 6: Cleanup +# Step 6: Summary # ============================================ -rm -f "$PLAYLISTS_JSON" "$ALL_TRACKS_JSON" "$TRACK_LOOKUP" - -# ============================================ -# Step 7: Summary -# ============================================ +PLAYLIST_COUNT=$(cat "$DATA_DIR/playlists.json" | jq '.playlists | length' 2>/dev/null || echo "0") +TRACK_COUNT=$(cat "$DATA_DIR/tracks.json" | jq 'length' 2>/dev/null || echo "0") echo "" -echo -e "${BLUE}========================================${NC}" -echo -e "${GREEN}📊 Playlist Summary${NC}" -echo -e "${BLUE}========================================${NC}" -echo -e " 📀 Playlists: $PLAYLIST_COUNT" -echo -e " 🎵 Total tracks found: $TRACK_COUNT" -echo -e " 📁 Data saved to: $DATA_DIR/" -echo -e "${BLUE}========================================${NC}" +echo "========================================" +echo "📊 Playlist Summary" +echo "========================================" +echo " 📀 Playlists: $PLAYLIST_COUNT" +echo " 🎵 Total tracks found: $TRACK_COUNT" +echo " 📁 Data saved to: $DATA_DIR/" +echo "========================================" -# Show playlist names echo "" -echo -e "${BLUE}📝 Playlist Names:${NC}" -cat "$OUTPUT_FILE" | $JQ_BIN -r '.playlists[].name' 2>/dev/null | while read -r name; do +echo "📝 Playlist Names:" +cat "$DATA_DIR/playlists.json" | jq -r '.playlists[].name' 2>/dev/null | while read -r name; do if [ -n "$name" ]; then echo " 🎵 $name" - else - echo " ⚠️ (unnamed)" fi done \ No newline at end of file diff --git a/generate-playlist-pages.sh b/generate-playlist-pages.sh index affb3f1..3ac88f9 100755 --- a/generate-playlist-pages.sh +++ b/generate-playlist-pages.sh @@ -13,6 +13,12 @@ if [ ! -f "$DATA_FILE" ]; then exit 1 fi +# Check if data file has content +if [ ! -s "$DATA_FILE" ]; then + echo "❌ Error: $DATA_FILE is empty. Run fetch-playlist.sh first." + exit 1 +fi + # Create content directory mkdir -p "$CONTENT_DIR" @@ -26,35 +32,63 @@ EOF fi # Clear old playlist pages (keep _index.md) +echo "Clearing old playlist pages..." find "$CONTENT_DIR" -maxdepth 1 -type f -name "*.md" ! -name "_index.md" -delete +echo "Generating playlist pages..." + +# Function to escape YAML strings +escape_yaml() { + echo "$1" | sed 's/"/\\"/g' | sed 's/:/\\:/g' | tr -d '\n\r' +} + # Process each playlist -cat "$DATA_FILE" | jq -c '.playlists[]' | while read -r playlist; do - NAME=$(echo "$playlist" | jq -r '.name // "Untitled Playlist"') - SLUG=$(echo "$NAME" | tr ' ' '-' | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]//g' | sed 's/--*/-/g' | sed 's/^-//;s/-$//') - - # If slug is empty, use the ID - if [ -z "$SLUG" ] || [ "$SLUG" = "untitled-playlist" ]; then - SLUG=$(echo "$playlist" | jq -r '.id' | head -c 12) +cat "$DATA_FILE" | jq -c '.playlists[]' 2>/dev/null | while read -r playlist; do + # Skip if playlist is empty + if [ -z "$playlist" ] || [ "$playlist" == "null" ]; then + continue fi + # Extract fields + NAME=$(echo "$playlist" | jq -r '.name // "Untitled Playlist"') + PLAYLIST_ID=$(echo "$playlist" | jq -r '.id // ""') DESCRIPTION=$(echo "$playlist" | jq -r '.description // ""') IMAGE=$(echo "$playlist" | jq -r '.image // ""') - TRACKS=$(echo "$playlist" | jq -c '.tracks') - PLAYLIST_ID=$(echo "$playlist" | jq -r '.id') - CREATED_AT=$(echo "$playlist" | jq -r '.created_at') + CREATED_AT=$(echo "$playlist" | jq -r '.created_at // 0') + TRACKS=$(echo "$playlist" | jq -c '.tracks // []') + + # Escape YAML strings + NAME_ESC=$(echo "$NAME" | sed 's/"/\\"/g') + DESCRIPTION_ESC=$(echo "$DESCRIPTION" | sed 's/"/\\"/g' | tr -d '\n\r') + IMAGE_ESC=$(echo "$IMAGE" | sed 's/"/\\"/g') + + # Skip if no name + if [ -z "$NAME" ] || [ "$NAME" == "null" ]; then + continue + fi + + # Create URL-friendly slug + SLUG=$(echo "$NAME" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/-/g' | sed 's/--*/-/g' | sed 's/^-//;s/-$//') + if [ -z "$SLUG" ]; then + SLUG="${PLAYLIST_ID:0:12}" + fi # Convert created_at to date - CREATED_DATE=$(date -d "@$CREATED_AT" "+%Y-%m-%d" 2>/dev/null || echo $(date "+%Y-%m-%d")) + if [ "$CREATED_AT" != "null" ] && [ "$CREATED_AT" -gt 0 ] 2>/dev/null; then + CREATED_DATE=$(date -d "@$CREATED_AT" "+%Y-%m-%d" 2>/dev/null || echo $(date "+%Y-%m-%d")) + else + CREATED_DATE=$(date "+%Y-%m-%d") + fi + # Write the markdown file with escaped values cat > "$CONTENT_DIR/$SLUG.md" < 🎵 {{ len $tracks }} tracks {{ if $created_at }} - 📅 {{ dateFormat "January 2, 2006" (time $created_at) }} + 📅 {{ dateFormat "January 2, 2006" (time $created_at) }} {{ end }}