Unable to fetch versions for a Cocoapods package

[morended]

The cocoapods gets the list of versions by parsing the html for the page where cocoapods specs are stored. Example : https://github.com/CocoaPods/Specs/tree/master/Specs/8/2/7/RandomKit

The html page is changed and hence the query selectors used to fetch version data no more exists on the page, as a result causing it to fail.

[scovetta]

Yep, looks like the GitHub page content changed, so maybe something like div.react-directory-truncate => title or something like that?

[gfs]

I wonder if we could make this more robust by switching to libgit2sharp rather than parsing the html representation of the git repo.

I'll test the workaround @scovetta suggested today first.

[gfs]

The version numbers are no longer included as actual HTML elements at all. Looking at the HTML that we get back it contains the below script tag with embedded data for react. I rewrote the parser to extract this json data, then parse it as a JObject, then use JSONPath to fetch just the version numbers, then we can add those to the version list. See #439.

<script type="application/json" data-target="react-app.embeddedData">{"payload":{"allShortcutsEnabled":false,"path":"Specs/8/2/7/RandomKit","repo":{"id":2365549,"defaultBranch":"master","name":"Specs","ownerLogin":"CocoaPods","currentUserCanPush":false,"isFork":false,"isEmpty":false,"createdAt":"2011-09-11T13:47:10.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/1189714?v=4","public":true,"private":false,"isOrgOwned":true},"currentUser":null,"refInfo":{"name":"master","listCacheKey":"v0:1613410512.605122","canEdit":false,"refType":"branch","currentOid":"ca3c3c1cc74a81418ff7a5ba96cea3b1e9cec9dd"},"tree":{"items":[{"name":"1.0.0","path":"Specs/8/2/7/RandomKit/1.0.0","contentType":"directory"},{"name":"1.1.0","path":"Specs/8/2/7/RandomKit/1.1.0","contentType":"directory"},{"name":"1.2.0","path":"Specs/8/2/7/RandomKit/1.2.0","contentType":"directory"},{"name":"1.3.0","path":"Specs/8/2/7/RandomKit/1.3.0","contentType":"directory"},{"name":"1.4.0","path":"Specs/8/2/7/RandomKit/1.4.0","contentType":"directory"},{"name":"1.5.0","path":"Specs/8/2/7/RandomKit/1.5.0","contentType":"directory"},{"name":"1.6.0","path":"Specs/8/2/7/RandomKit/1.6.0","contentType":"directory"},{"name":"2.0.0","path":"Specs/8/2/7/RandomKit/2.0.0","contentType":"directory"},{"name":"2.1.0","path":"Specs/8/2/7/RandomKit/2.1.0","contentType":"directory"},{"name":"2.2.0","path":"Specs/8/2/7/RandomKit/2.2.0","contentType":"directory"},{"name":"2.2.1","path":"Specs/8/2/7/RandomKit/2.2.1","contentType":"directory"},{"name":"2.3.0","path":"Specs/8/2/7/RandomKit/2.3.0","contentType":"directory"},{"name":"3.0.0","path":"Specs/8/2/7/RandomKit/3.0.0","contentType":"directory"},{"name":"4.0.0","path":"Specs/8/2/7/RandomKit/4.0.0","contentType":"directory"},{"name":"4.1.0","path":"Specs/8/2/7/RandomKit/4.1.0","contentType":"directory"},{"name":"4.2.0","path":"Specs/8/2/7/RandomKit/4.2.0","contentType":"directory"},{"name":"4.3.0","path":"Specs/8/2/7/RandomKit/4.3.0","contentType":"directory"},{"name":"4.3.1","path":"Specs/8/2/7/RandomKit/4.3.1","contentType":"directory"},{"name":"4.4.0","path":"Specs/8/2/7/RandomKit/4.4.0","contentType":"directory"},{"name":"4.4.1","path":"Specs/8/2/7/RandomKit/4.4.1","contentType":"directory"},{"name":"4.5.0","path":"Specs/8/2/7/RandomKit/4.5.0","contentType":"directory"},{"name":"4.5.1","path":"Specs/8/2/7/RandomKit/4.5.1","contentType":"directory"},{"name":"4.5.2","path":"Specs/8/2/7/RandomKit/4.5.2","contentType":"directory"},{"name":"5.0.0","path":"Specs/8/2/7/RandomKit/5.0.0","contentType":"directory"},{"name":"5.1.0","path":"Specs/8/2/7/RandomKit/5.1.0","contentType":"directory"},{"name":"5.2.0","path":"Specs/8/2/7/RandomKit/5.2.0","contentType":"directory"},{"name":"5.2.1","path":"Specs/8/2/7/RandomKit/5.2.1","contentType":"directory"},{"name":"5.2.2","path":"Specs/8/2/7/RandomKit/5.2.2","contentType":"directory"},{"name":"5.2.3","path":"Specs/8/2/7/RandomKit/5.2.3","contentType":"directory"}],"templateDirectorySuggestionUrl":null,"readme":null,"totalCount":29,"showBranchInfobar":false},"fileTree":{"Specs/8/2/7":{"items":[{"name":"BBUtils","path":"Specs/8/2/7/BBUtils","contentType":"directory"},{"name":"BadgeLabel","path":"Specs/8/2/7/BadgeLabel","contentType":"directory"},{"name":"EMSpinnerButton","path":"Specs/8/2/7/EMSpinnerButton","contentType":"directory"},{"name":"ExpandableTextView","path":"Specs/8/2/7/ExpandableTextView","contentType":"directory"},{"name":"FCModel","path":"Specs/8/2/7/FCModel","contentType":"directory"},{"name":"FSDiary","path":"Specs/8/2/7/FSDiary","contentType":"directory"},{"name":"JBTestZhenNiuB","path":"Specs/8/2/7/JBTestZhenNiuB","contentType":"directory"},{"name":"JavenKit","path":"Specs/8/2/7/JavenKit","contentType":"directory"},{"name":"NSURL-RandValue","path":"Specs/8/2/7/NSURL-RandValue","contentType":"directory"},{"name":"NativeCSS","path":"Specs/8/2/7/NativeCSS","contentType":"directory"},{"name":"PRTabWithBadgeAnimation","path":"Specs/8/2/7/PRTabWithBadgeAnimation","contentType":"directory"},{"name":"RandomKit","path":"Specs/8/2/7/RandomKit","contentType":"directory"},{"name":"SNKeyboard","path":"Specs/8/2/7/SNKeyboard","contentType":"directory"},{"name":"Scheme","path":"Specs/8/2/7/Scheme","contentType":"directory"},{"name":"SobotCallSDK","path":"Specs/8/2/7/SobotCallSDK","contentType":"directory"},{"name":"SpringButton","path":"Specs/8/2/7/SpringButton","contentType":"directory"},{"name":"TrackingHelper","path":"Specs/8/2/7/TrackingHelper","contentType":"directory"},{"name":"YUGLTransition","path":"Specs/8/2/7/YUGLTransition","contentType":"directory"},{"name":"ZDAFNetWorkHelper","path":"Specs/8/2/7/ZDAFNetWorkHelper","contentType":"directory"},{"name":"yapo-transport","path":"Specs/8/2/7/yapo-transport","contentType":"directory"}],"totalCount":20},"Specs/8/2":{"items":[{"name":"0","path":"Specs/8/2/0","contentType":"directory"},{"name":"1","path":"Specs/8/2/1","contentType":"directory"},{"name":"2","path":"Specs/8/2/2","contentType":"directory"},{"name":"3","path":"Specs/8/2/3","contentType":"directory"},{"name":"4","path":"Specs/8/2/4","contentType":"directory"},{"name":"5","path":"Specs/8/2/5","contentType":"directory"},{"name":"6","path":"Specs/8/2/6","contentType":"directory"},{"name":"7","path":"Specs/8/2/7","contentType":"directory"},{"name":"8","path":"Specs/8/2/8","contentType":"directory"},{"name":"9","path":"Specs/8/2/9","contentType":"directory"},{"name":"a","path":"Specs/8/2/a","contentType":"directory"},{"name":"b","path":"Specs/8/2/b","contentType":"directory"},{"name":"c","path":"Specs/8/2/c","contentType":"directory"},{"name":"d","path":"Specs/8/2/d","contentType":"directory"},{"name":"e","path":"Specs/8/2/e","contentType":"directory"},{"name":"f","path":"Specs/8/2/f","contentType":"directory"}],"totalCount":16},"Specs/8":{"items":[{"name":"0","path":"Specs/8/0","contentType":"directory"},{"name":"1","path":"Specs/8/1","contentType":"directory"},{"name":"2","path":"Specs/8/2","contentType":"directory"},{"name":"3","path":"Specs/8/3","contentType":"directory"},{"name":"4","path":"Specs/8/4","contentType":"directory"},{"name":"5","path":"Specs/8/5","contentType":"directory"},{"name":"6","path":"Specs/8/6","contentType":"directory"},{"name":"7","path":"Specs/8/7","contentType":"directory"},{"name":"8","path":"Specs/8/8","contentType":"directory"},{"name":"9","path":"Specs/8/9","contentType":"directory"},{"name":"a","path":"Specs/8/a","contentType":"directory"},{"name":"b","path":"Specs/8/b","contentType":"directory"},{"name":"c","path":"Specs/8/c","contentType":"directory"},{"name":"d","path":"Specs/8/d","contentType":"directory"},{"name":"e","path":"Specs/8/e","contentType":"directory"},{"name":"f","path":"Specs/8/f","contentType":"directory"}],"totalCount":16},"Specs":{"items":[{"name":"0","path":"Specs/0","contentType":"directory"},{"name":"1","path":"Specs/1","contentType":"directory"},{"name":"2","path":"Specs/2","contentType":"directory"},{"name":"3","path":"Specs/3","contentType":"directory"},{"name":"4","path":"Specs/4","contentType":"directory"},{"name":"5","path":"Specs/5","contentType":"directory"},{"name":"6","path":"Specs/6","contentType":"directory"},{"name":"7","path":"Specs/7","contentType":"directory"},{"name":"8","path":"Specs/8","contentType":"directory"},{"name":"9","path":"Specs/9","contentType":"directory"},{"name":"a","path":"Specs/a","contentType":"directory"},{"name":"b","path":"Specs/b","contentType":"directory"},{"name":"c","path":"Specs/c","contentType":"directory"},{"name":"d","path":"Specs/d","contentType":"directory"},{"name":"e","path":"Specs/e","contentType":"directory"},{"name":"f","path":"Specs/f","contentType":"directory"},{"name":".gitkeep","path":"Specs/.gitkeep","contentType":"file"}],"totalCount":17},"":{"items":[{"name":".github","path":".github","contentType":"directory"},{"name":"Scripts","path":"Scripts","contentType":"directory"},{"name":"Specs","path":"Specs","contentType":"directory"},{"name":".gitignore","path":".gitignore","contentType":"file"},{"name":".rubocop.yml","path":".rubocop.yml","contentType":"file"},{"name":"AlgoliaSearch.yml","path":"AlgoliaSearch.yml","contentType":"file"},{"name":"CocoaPods-version.yml","path":"CocoaPods-version.yml","contentType":"file"},{"name":"Gemfile","path":"Gemfile","contentType":"file"},{"name":"Gemfile.lock","path":"Gemfile.lock","contentType":"file"},{"name":"README.md","path":"README.md","contentType":"file"},{"name":"netlify.toml","path":"netlify.toml","contentType":"file"}],"totalCount":11}},"fileTreeProcessingTime":30.031953,"foldersToFetch":[],"showSurveyBanner":false,"showCodeNavSurvey":false,"csrf_tokens":{"/CocoaPods/Specs/branches":{"post":"3io837AB6d-fdf5TCaeBvZefdJ61sK2UvJOxT0KrwWhgDrpBgYMXFIQHy70GH1baVoxdScj02QvhWeXf_ZwDhA"},"/CocoaPods/Specs/branches/fetch_and_merge/master":{"post":"BgJKof6DgIhiHYBr0eYoIpONxbTTlKe0XhUVsUvksrJg8laVxfHqr9Wsm5StinZtrfv6LsuKWso0s0g30FxAkA"},"/CocoaPods/Specs/branches/fetch_and_merge/master?discard_changes=true":{"post":"qkw7zAALxLXyaPL44l0V4kDgLeFYfB45zulz26fsHp_MvCf4O3mukkXZ6QeeMUutfpYSe0Bi40ekTy5dPFTsvQ"}}},"title":"Specs/Specs/8/2/7/RandomKit at master · CocoaPods/Specs","locale":"en","appPayload":{"helpUrl":"https://docs.github.com","findFileWorkerPath":"/assets-cdn/worker/find-file-worker-848bb9a5da17.js","findInFileWorkerPath":"/assets-cdn/worker/find-in-file-worker-5155f0e7ee32.js","githubDevUrl":null,"enabled_features":{"virtualize_file_tree":true,"react_repos_overview":false,"repos_new_shortcut_enabled":true,"blob_navigation_cursor":true,"code_nav_ui_events":false,"ref_selector_v2":false,"codeview_codemirror_next_editors":false,"blob_firefox_separate_characters":true,"column_symbol_auto_select":false,"copilot_conversational_ux":false}}}</script>