diff --git a/lib/docs/filters/homebrew/entries.rb b/lib/docs/filters/homebrew/entries.rb index b3a3ba1826..2c355d7d07 100644 --- a/lib/docs/filters/homebrew/entries.rb +++ b/lib/docs/filters/homebrew/entries.rb @@ -3,8 +3,20 @@ class Homebrew class EntriesFilter < Docs::EntriesFilter def get_name header = at_css('h1') - name = header.nil? ? current_url.normalized_path[1..-1].gsub(/-/, ' ') : header.content.strip - name.remove! %r{\(.*} + + if current_url.path.start_with?('/rubydoc') + # Titles in rubydoc content follow a different format, with a bit of cleanup needed. + name = header + .content + .gsub('Class: ', '') + .gsub('Module: ', '') + .gsub('Exception: ', '') + .gsub(' Private', '') + else + name = header.nil? ? current_url.normalized_path[1..-1].gsub(/-/, ' ') : header.content.strip + name.remove! %r{\(.*} + end + name end @@ -26,13 +38,21 @@ def get_name Building-Against-Non-Homebrew-Dependencies How-to-Create-and-Maintain-a-Tap Brew-Test-Bot - Prose-Style-Guidelines Typechecking Reproducible-Builds ) def get_type - if CONTRIBUTOR_SLUGS.include?(slug) + if current_url.path.start_with?('/rubydoc') + header = at_css('h1').content + if header.start_with?('Module') + 'API Modules' + elsif header.start_with?('Class') + 'API Classes' + else + 'API Exceptions' + end + elsif CONTRIBUTOR_SLUGS.include?(slug) 'Contributors' else 'Users' diff --git a/lib/docs/scrapers/haskell.rb b/lib/docs/scrapers/haskell.rb index 8d822ece0f..d0aeda1456 100644 --- a/lib/docs/scrapers/haskell.rb +++ b/lib/docs/scrapers/haskell.rb @@ -78,10 +78,9 @@ class Haskell < UrlScraper end def get_latest_version(opts) - doc = fetch_doc('https://www.haskell.org/ghc/download.html', opts) - links = doc.css('a').to_a - versions = links.map {|link| link.content.scan(/\A([0-9.]+)\Z/)} - versions.find {|version| !version.empty?}[0][0] + tags = get_github_tags('ghc', 'ghc', opts) + tag = tags.find {|t| t['name'].ends_with?('-release') }['name'] + tag[/ghc-(.*)-release/, 1] end end diff --git a/lib/docs/scrapers/homebrew.rb b/lib/docs/scrapers/homebrew.rb index 0faeaa5785..353ec01d2b 100644 --- a/lib/docs/scrapers/homebrew.rb +++ b/lib/docs/scrapers/homebrew.rb @@ -1,8 +1,10 @@ module Docs class Homebrew < UrlScraper + # include MultipleBaseUrls + self.name = 'Homebrew' self.type = 'simple' - self.release = '4.6.15' + self.release = '6.0.3' self.base_url = 'https://docs.brew.sh/' self.links = { home: 'https://brew.sh', @@ -11,9 +13,29 @@ class Homebrew < UrlScraper html_filters.push 'homebrew/entries', 'homebrew/clean_html' - options[:container] = ->(filter) { filter.root_page? ? '#home' : '#page' } + options[:container] = ->(filter) do + if filter.current_url.path.include?('rubydoc') + '#content' + else + '#default' + end + end - options[:skip_patterns] = [/maintainer/i, /core\-contributor/i, /kickstarter/i, /governance/i] + # Mostly handbook articles with no code. + options[:skip_patterns] = [ + /maintainer/i, + /core\-contributor/i, + /kickstarter/i, + /governance/i, + /responsible/i, + /leadership/i, + /prose/i, + /expense/i, + /creating-a-homebrew-issue/i, + /homebrew-and-java/i, + /checksum_deprecation/i, + /Working-with/i + ] options[:attribution] = <<-HTML © 2009–present Homebrew contributors
diff --git a/lib/docs/scrapers/love.rb b/lib/docs/scrapers/love.rb index f702da8be9..1979db059a 100644 --- a/lib/docs/scrapers/love.rb +++ b/lib/docs/scrapers/love.rb @@ -43,8 +43,7 @@ class Love < UrlScraper HTML def get_latest_version(opts) - doc = fetch_doc('https://love2d.org/wiki/Version_History', opts) - doc.at_css('#mw-content-text table a').content + get_github_tags('love2d', 'love', opts).first['name'] end end end diff --git a/lib/docs/scrapers/mongoose.rb b/lib/docs/scrapers/mongoose.rb index bcd9673329..1818c78c59 100644 --- a/lib/docs/scrapers/mongoose.rb +++ b/lib/docs/scrapers/mongoose.rb @@ -29,9 +29,7 @@ class Mongoose < UrlScraper HTML def get_latest_version(opts) - doc = fetch_doc('https://mongoosejs.com/docs/', opts) - label = doc.at_css('.pure-menu-link').content.strip - label.sub(/Version /, '') + get_github_tags('Automattic', 'mongoose', opts).find {|t| t['name'].starts_with?(/\d/)}['name'] end end end diff --git a/lib/docs/scrapers/python.rb b/lib/docs/scrapers/python.rb index 7c80e7549a..a0bd1ee480 100644 --- a/lib/docs/scrapers/python.rb +++ b/lib/docs/scrapers/python.rb @@ -15,6 +15,7 @@ class Python < UrlScraper options[:skip_patterns] = [/whatsnew/] options[:skip] = %w( + archives/ library/2to3.html library/formatter.html library/intro.html @@ -28,7 +29,7 @@ class Python < UrlScraper HTML version '3.14' do - self.release = '3.14.4' + self.release = '3.14.6' self.base_url = "https://docs.python.org/#{self.version}/" html_filters.push 'python/entries_v3', 'sphinx/clean_html', 'python/clean_html' diff --git a/lib/docs/scrapers/tensorflow/tensorflow.rb b/lib/docs/scrapers/tensorflow/tensorflow.rb index 770d35b87c..a54e2ac014 100644 --- a/lib/docs/scrapers/tensorflow/tensorflow.rb +++ b/lib/docs/scrapers/tensorflow/tensorflow.rb @@ -45,8 +45,8 @@ class Tensorflow < UrlScraper end def get_latest_version(opts) - doc = fetch_doc(self.base_url, opts) - doc.title[/TensorFlow v([.\d]+)/, 1] + tag = get_github_tags('tensorflow', 'tensorflow', opts).find { |t| !t['name'].include?('-rc') } + tag['name'][1..-1] end private diff --git a/lib/docs/scrapers/threejs.rb b/lib/docs/scrapers/threejs.rb index c3ef9bc25d..7fd3147359 100644 --- a/lib/docs/scrapers/threejs.rb +++ b/lib/docs/scrapers/threejs.rb @@ -44,17 +44,17 @@ class Threejs < UrlScraper self.base_url = "https://threejs.org/docs" def get_latest_version(opts) - get_latest_github_release('mrdoob', 'three.js', opts)[1..] + get_github_tags('mrdoob', 'three.js', opts).first['name'][1..] end def initial_paths paths = [] - url = 'https://threejs.org/docs/list.json' + url = 'https://threejs.org/docs/search.json' response = Request.run(url) json_data = JSON.parse(response.body) # Process both API and manual sections - process_documentation(json_data['en'], paths) + process_documentation(json_data, paths) paths end @@ -78,4 +78,4 @@ def process_documentation(data, paths, prefix = '') end end end -end \ No newline at end of file +end