sourceforge-scraper/sourceforge_scraper.sh at main · milahu/sourceforge-scraper · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
#! /usr/bin/env bash

debug=false
#debug=true

function sourceforge_get_file_urls() {

  # return one folder per line, multiple file urls per line

  local folder_url="$1"

  local html="$(curl -s "$folder_url")"

  local links="$(echo "$html" | grep '^<th scope="row" headers="files_name_h"><a href' | cut -d'"' -f6 | sed 's|^/|https://sourceforge.net/|')"

  $debug && echo "sourceforge_get_file_urls: links:" >&2
  $debug && echo "$links" | sed 's/^/  /' >&2

  # loop files
  # convert file urls to "proper basename" urls = locally resolve the first http redirect
  # example:
  # a: https://          sourceforge.net/projects/sevenzip/files/7-Zip/21.07/7z2107-src.tar.xz/download
  # b: https://downloads.sourceforge.net/project /sevenzip      /7-Zip/21.07/7z2107-src.tar.xz
  $debug && echo "sourceforge_get_file_urls: looping files" >&2
  file_links=$(echo "$links" | grep '/download$')
  if [ -n "$file_links" ]; then
    echo "$file_links" |
    sed -E 's|^https://sourceforge.net/projects/([^/]+)/files/([^/]+)/(.*?)/download$|https://downloads.sourceforge.net/project/\1/\2/\3|' |
    xargs echo -n
    echo
  fi

  # loop folders
  $debug && echo "sourceforge_get_file_urls: looping folders" >&2
  while read next_folder_url; do
    # recurse
    $debug && echo "sourceforge_get_file_urls: recurse from $folder_url to $next_folder_url" >&2
    sourceforge_get_file_urls "$next_folder_url"
  done < <(echo "$links" | grep '/$')
}


# generic: get url from args, download all files

if [ -n "$1" ]; then
  exec sourceforge_get_file_urls "$1" |
  xargs -n1 wget --no-clobber --recursive --level=1
fi


# example: get only the 7z source tarballs from the sevenzip project

echo example: sourceforge_get_file_urls https://sourceforge.net/projects/sevenzip/files/7-Zip/ 7zip-files
sourceforge_get_file_urls https://sourceforge.net/projects/sevenzip/files/7-Zip/ 7zip-files |
while read folder_urls; do
  folder_urls=$(echo "$folder_urls" | tr ' ' $'\n')
  $debug && echo -e "folder_urls:\n$folder_urls"
  # pick only one file per folder
  file_url="$folder_urls"
  file_url=$(echo "$file_url" | grep -E '/7z[0-9]+(-src\.tar\.xz|-src\.7z|\.tar\.bz2)$')
  if [[ "$(echo "$file_url" | wc -l)" != 1 ]]; then
    # we still have multiple file urls
    # remove *.7z
    # example file urls:
    # https://downloads.sourceforge.net/project/sevenzip/7-Zip/22.01/7z2201-src.tar.xz
    # https://downloads.sourceforge.net/project/sevenzip/7-Zip/22.01/7z2201-src.7z
    file_url=$(echo "$file_url" | grep -E '/7z[0-9]+(-src\.tar\.xz|\.tar\.bz2)$')
  fi
  if [[ "$(echo "$file_url" | wc -l)" != 1 ]]; then
    # we still have multiple file urls
    echo "FIXME filter file urls:" >&2
    echo "$file_url" | sed 's/^/  /' >&2
  fi
  if [[ "$(echo "$file_url" | wc -l)" == 1 ]]; then
    $debug && echo "ok: $file_url" >&2
  fi
  echo "$file_url"
done |
xargs -n1 wget --no-clobber --recursive --level=1

# note: "wget --no-clobber" assumes that existing files are valid

# list all downloaded files, sort by version
echo "downloaded files:"
find downloads.sourceforge.net/ -type f | sort -V | sed 's/^/  /'