+67

lib/cargo_import.rb

··· 1 1 + require_relative 'requests' 2 2 + require 'json' 3 3 + require 'time' 4 4 + 5 5 + class CargoImport 6 6 + include Requests 7 7 + 8 8 + class CargoToml 9 9 + def initialize(contents) 10 10 + @contents = contents 11 11 + end 12 12 + 13 13 + def name 14 14 + return unless package_section 15 15 + 16 16 + name_line = package_section.match(/^\s*name\s*=\s*"([^"]+)"/) 17 17 + name_line && name_line[1] 18 18 + end 19 19 + 20 20 + def package_section 21 21 + @package_section ||= begin 22 22 + match = @contents.match(/^\[package\](.+?)(^\[.+\]|\z)/m) 23 23 + match && match[1] 24 24 + end 25 25 + end 26 26 + end 27 27 + 28 28 + class RemoteCrate 29 29 + def initialize(json) 30 30 + @json = json 31 31 + end 32 32 + 33 33 + def name 34 34 + @json['crate']['name'] 35 35 + end 36 36 + 37 37 + def last_release_time 38 38 + @json['versions'].map { |v| Time.parse(v['created_at']) }.sort.last 39 39 + end 40 40 + 41 41 + def homepage_url 42 42 + @json['crate']['homepage'] 43 43 + end 44 44 + 45 45 + def repository_url 46 46 + @json['crate']['repository'] 47 47 + end 48 48 + end 49 49 + 50 50 + def initialize 51 51 + @@cache ||= {} 52 52 + end 53 53 + 54 54 + def get_crate_info(name) 55 55 + sleep 1 56 56 + response = @@cache[name] || get_response("https://crates.io/api/v1/crates/#{name}") 57 57 + @@cache[name] = response 58 58 + 59 59 + if response.code.to_i == 200 60 60 + RemoteCrate.new(JSON.parse(response.body)) 61 61 + elsif response.code.to_i == 404 62 62 + nil 63 63 + else 64 64 + raise FetchError.new(response) 65 65 + end 66 66 + end 67 67 + end

+42

lib/github_import.rb

··· 1 1 + require_relative 'cargo_import' 1 2 require_relative 'import_helpers' 2 3 require_relative 'npm_import' 3 4 require_relative 'requests' ··· 39 40 if ['JavaScript', 'TypeScript'].include?(repo_info['language']) 40 41 npm = get_npm_releases(user, repo, project) 41 42 last_update = npm.map { |n| n.last_release_time }.sort.last 43 43 + 44 44 + if last_update && (release.nil? || last_update > release['published_at']) 45 45 + data['last_release'] = { 'published_at' => last_update } 46 46 + end 47 47 + elsif repo_info['language'] == 'Rust' 48 48 + crates = get_cargo_releases(user, repo, project) 49 49 + last_update = crates.map { |c| c.last_release_time }.sort.last 42 50 43 51 if last_update && (release.nil? || last_update > release['published_at']) 44 52 data['last_release'] = { 'published_at' => last_update } ··· 173 181 174 182 if project.urls.map { |u| normalize_repo_url(u) }.any? { |u| u == package_repo_url || u == package_homepage } 175 183 releases << npm 184 184 + end 185 185 + end 186 186 + end 187 187 + 188 188 + releases 189 189 + end 190 190 + 191 191 + def get_cargo_releases(user, repo, project) 192 192 + sleep 5 193 193 + 194 194 + search_url = URI("https://api.github.com/search/code") 195 195 + search_url.query = URI.encode_www_form(q: "repo:#{user}/#{repo} filename:Cargo.toml", per_page: 100) 196 196 + response = get_response(search_url) 197 197 + raise FetchError.new(response) unless response.code.to_i == 200 198 198 + 199 199 + json = JSON.parse(response.body) 200 200 + releases = [] 201 201 + 202 202 + json['items'].each do |file| 203 203 + response = get_response(file['url']) 204 204 + raise FetchError.new(response) unless response.code.to_i == 200 205 205 + 206 206 + details = JSON.parse(response.body) 207 207 + contents = Base64.decode64(details['content']) 208 208 + package = CargoImport::CargoToml.new(contents) 209 209 + 210 210 + next if package.name.nil? 211 211 + 212 212 + if crate = CargoImport.new.get_crate_info(package.name) 213 213 + crate_repo_url = normalize_repo_url(crate.repository_url) 214 214 + crate_homepage = normalize_repo_url(crate.homepage_url) 215 215 + 216 216 + if project.urls.map { |u| normalize_repo_url(u) }.any? { |u| u == crate_repo_url || u == crate_homepage } 217 217 + releases << crate 176 218 end 177 219 end 178 220 end

+33

lib/tangled_import.rb

··· 1 1 + require_relative 'cargo_import' 1 2 require_relative 'import_helpers' 2 3 require_relative 'npm_import' 3 4 require_relative 'requests' ··· 53 54 if project.language == :js 54 55 npm = get_npm_releases(repo_folder, project) 55 56 last_update = npm.map { |n| n.last_release_time }.sort.last 57 57 + 58 58 + if last_update 59 59 + data['last_release'] = { 'published_at' => last_update } 60 60 + end 61 61 + elsif project.language == :rust 62 62 + crates = get_cargo_releases(repo_folder, project) 63 63 + last_update = crates.map { |c| c.last_release_time }.sort.last 56 64 57 65 if last_update 58 66 data['last_release'] = { 'published_at' => last_update } ··· 143 151 144 152 if project.urls.map { |u| normalize_repo_url(u) }.any? { |u| u == package_repo_url || u == package_homepage } 145 153 releases << npm 154 154 + end 155 155 + end 156 156 + end 157 157 + 158 158 + releases 159 159 + end 160 160 + end 161 161 + 162 162 + def get_cargo_releases(repo_folder, project) 163 163 + Dir.chdir(repo_folder) do 164 164 + packages = %x(find . -name 'Cargo.toml').strip 165 165 + releases = [] 166 166 + 167 167 + packages.lines.each do |file| 168 168 + contents = File.read(file.strip) 169 169 + package = CargoImport::CargoToml.new(contents) 170 170 + 171 171 + next if package.name.nil? 172 172 + 173 173 + if crate = CargoImport.new.get_crate_info(package.name) 174 174 + crate_repo_url = normalize_repo_url(crate.repository_url) 175 175 + crate_homepage = normalize_repo_url(crate.homepage_url) 176 176 + 177 177 + if project.urls.map { |u| normalize_repo_url(u) }.any? { |u| u == crate_repo_url || u == crate_homepage } 178 178 + releases << crate 146 179 end 147 180 end 148 181 end