https://github.com/sdq/kaggle-crawler

simple scrapy project for kaggle.com
https://github.com/sdq/kaggle-crawler

crawler kaggle

Last synced: about 1 month ago
JSON representation

simple scrapy project for kaggle.com

Host: GitHub
URL: https://github.com/sdq/kaggle-crawler
Owner: sdq
License: mit
Created: 2017-09-13T08:27:54.000Z (almost 9 years ago)
Default Branch: master
Last Pushed: 2017-09-13T08:43:21.000Z (almost 9 years ago)
Last Synced: 2025-08-21T11:52:32.920Z (10 months ago)
Topics: crawler, kaggle
Language: Python
Homepage:
Size: 24.4 KB
Stars: 8
Watchers: 1
Forks: 0
Open Issues: 0
Metadata Files:
- Readme: README.md
- License: LICENSE

Awesome Lists containing this project

README

# kaggle-crawler

```python
scrapy crawl competitions
scrapy crawl datasets
scrapy crawl kernels
scrapy crawl discussion
```

competitions result demo
```json
{
"sortByOptions": [
{
"value": "prize",
"label": "Prize"
},
{
"value": "deadline",
"label": "Deadline"
},
{
"value": "numberOfTeams",
"label": "Number of teams"
},
{
"value": "relevance",
"label": "Relevance"
},
{
"value": "recentlyCreated",
"label": "Recently created"
}
],
"selectionGroupOptions": [
{
"value": "active",
"label": "Active"
},
{
"value": "all",
"label": "All"
},
{
"value": "entered",
"label": "Entered"
}
],
"hostSegmentOptions": [
{
"value": "",
"label": "All Categories"
},
{
"value": "featured",
"label": "Featured"
},
{
"value": "research",
"label": "Research"
},
{
"value": "recruitment",
"label": "Recruitment"
},
{
"value": "gettingStarted",
"label": "Getting Started"
},
{
"value": "playground",
"label": "Playground"
},
{
"value": "inClass",
"label": "InClass"
}
],
"totalCompetitions": 17,
"competitions": [
{
"competitionId": 6775,
"competitionTitle": "Passenger Screening Algorithm Challenge",
"competitionDescription": "Improve the accuracy of the Department of Homeland Security\\u0027s threat recognition algorithms",
"competitionUrl": "/c/passenger-screening-algorithm-challenge",
"thumbnailImageUrl": "https://kaggle2.blob.core.windows.net/competitions/kaggle/6775/logos/thumb76_76.png",
"deadline": "2017-12-16T07:59:00Z",
"totalTeams": 164,
"totalKernels": 0,
"rewardQuantity": 1500000,
"rewardTypeName": "USD",
"organizationName": "Department of Homeland Security",
"organizationUrl": "dhs",
"hostSegment": "featured",
"isLimited": false,
"isPrivate": false,
"isInClass": false,
"userHasEntered": false,
"rewardDisplay": "$1,500,000"
},
{
"competitionId": 6649,
"competitionTitle": "Zillow Prize: Zillow’s Home Value Prediction (Zestimate)",
"competitionDescription": "Can you improve the algorithm that changed the world of real estate?",
"competitionUrl": "/c/zillow-prize-1",
"thumbnailImageUrl": "https://kaggle2.blob.core.windows.net/competitions/kaggle/6649/logos/thumb76_76.png",
"deadline": "2018-01-17T15:59:00Z",
"totalTeams": 2022,
"totalKernels": 0,
"rewardQuantity": 1200000,
"rewardTypeName": "USD",
"organizationName": "Zillow",
"organizationUrl": "zillow",
"hostSegment": "featured",
"isLimited": false,
"isPrivate": false,
"isInClass": false,
"userHasEntered": false,
"rewardDisplay": "$1,200,000"
},
{
"competitionId": 6927,
"competitionTitle": "Carvana Image Masking Challenge",
"competitionDescription": "Automatically identify the boundaries of the car in an image",
"competitionUrl": "/c/carvana-image-masking-challenge",
"thumbnailImageUrl": "https://kaggle2.blob.core.windows.net/competitions/kaggle/6927/logos/thumb76_76.png",
"deadline": "2017-09-27T23:59:00Z",
"totalTeams": 249,
"totalKernels": 0,
"rewardQuantity": 25000,
"rewardTypeName": "USD",
"organizationName": "Carvana",
"organizationUrl": "carvana",
"hostSegment": "featured",
"isLimited": false,
"isPrivate": false,
"isInClass": false,
"userHasEntered": false,
"rewardDisplay": "$25,000"
},
{
"competitionId": 6644,
"competitionTitle": "Instacart Market Basket Analysis",
"competitionDescription": "Which products will an Instacart consumer purchase again?",
"competitionUrl": "/c/instacart-market-basket-analysis",
"thumbnailImageUrl": "https://kaggle2.blob.core.windows.net/competitions/kaggle/6644/logos/thumb76_76.png",
"deadline": "2017-08-14T23:59:00Z",
"totalTeams": 2547,
"totalKernels": 0,
"rewardQuantity": 25000,
"rewardTypeName": "USD",
"organizationName": "Instacart",
"organizationUrl": "instacart",
"hostSegment": "featured",
"isLimited": false,
"isPrivate": false,
"isInClass": false,
"userHasEntered": false,
"rewardDisplay": "$25,000"
},
{
"competitionId": 6768,
"competitionTitle": "Web Traffic Time Series Forecasting",
"competitionDescription": "Forecast future traffic to Wikipedia pages",
"competitionUrl": "/c/web-traffic-time-series-forecasting",
"thumbnailImageUrl": "https://kaggle2.blob.core.windows.net/competitions/kaggle/6768/logos/thumb76_76.png",
"deadline": "2017-09-10T23:59:00Z",
"totalTeams": 530,
"totalKernels": 0,
"rewardQuantity": 25000,
"rewardTypeName": "USD",
"organizationName": "Google",
"organizationUrl": "google",
"hostSegment": "research",
"isLimited": false,
"isPrivate": false,
"isInClass": false,
"userHasEntered": false,
"rewardDisplay": "$25,000"
},
{
"competitionId": 6841,
"competitionTitle": "Personalized Medicine: Redefining Cancer Treatment",
"competitionDescription": "Predict the effect of Genetic Variants to enable Personalized Medicine",
"competitionUrl": "/c/msk-redefining-cancer-treatment",
"thumbnailImageUrl": "https://kaggle2.blob.core.windows.net/competitions/kaggle/6841/logos/thumb76_76.png",
"deadline": "2017-10-03T06:59:00Z",
"totalTeams": 705,
"totalKernels": 0,
"rewardQuantity": 15000,
"rewardTypeName": "USD",
"organizationName": null,
"organizationUrl": null,
"hostSegment": "research",
"isLimited": false,
"isPrivate": false,
"isInClass": false,
"userHasEntered": false,
"rewardDisplay": "$15,000"
},
{
"competitionId": 6864,
"competitionTitle": "NIPS 2017: Non-targeted Adversarial Attack",
"competitionDescription": "Imperceptibly transform images in ways that fool classification models",
"competitionUrl": "/c/nips-2017-non-targeted-adversarial-attack",
"thumbnailImageUrl": "https://kaggle2.blob.core.windows.net/competitions/kaggle/6864/logos/thumb76_76.png",
"deadline": "2017-10-01T23:59:00Z",
"totalTeams": 0,
"totalKernels": 0,
"rewardQuantity": 0,
"rewardTypeName": "Swag",
"organizationName": "Google Brain",
"organizationUrl": "google-brain",
"hostSegment": "research",
"isLimited": false,
"isPrivate": false,
"isInClass": false,
"userHasEntered": false,
"rewardDisplay": "Swag"
},
{
"competitionId": 6866,
"competitionTitle": "NIPS 2017: Targeted Adversarial Attack",
"competitionDescription": "Develop an adversarial attack that causes image classifiers to predict a specific target class",
"competitionUrl": "/c/nips-2017-targeted-adversarial-attack",
"thumbnailImageUrl": "https://kaggle2.blob.core.windows.net/competitions/kaggle/6866/logos/thumb76_76.png",
"deadline": "2017-10-01T23:59:00Z",
"totalTeams": 0,
"totalKernels": 0,
"rewardQuantity": 0,
"rewardTypeName": "Swag",
"organizationName": "Google Brain",
"organizationUrl": "google-brain",
"hostSegment": "research",
"isLimited": false,
"isPrivate": false,
"isInClass": false,
"userHasEntered": false,
"rewardDisplay": "Swag"
},
{
"competitionId": 6867,
"competitionTitle": "NIPS 2017: Defense Against Adversarial Attack",
"competitionDescription": "Create an image classifier that is robust to adversarial attacks",
"competitionUrl": "/c/nips-2017-defense-against-adversarial-attack",
"thumbnailImageUrl": "https://kaggle2.blob.core.windows.net/competitions/kaggle/6867/logos/thumb76_76.png",
"deadline": "2017-10-01T23:59:00Z",
"totalTeams": 0,
"totalKernels": 0,
"rewardQuantity": 0,
"rewardTypeName": "Swag",
"organizationName": "Google Brain",
"organizationUrl": "google-brain",
"hostSegment": "research",
"isLimited": false,
"isPrivate": false,
"isInClass": false,
"userHasEntered": false,
"rewardDisplay": "Swag"
},
{
"competitionId": 6799,
"competitionTitle": "ImageNet Object Localization Challenge",
"competitionDescription": "Identify the objects in images",
"competitionUrl": "/c/imagenet-object-localization-challenge",
"thumbnailImageUrl": "https://kaggle2.blob.core.windows.net/competitions/kaggle/6796/logos/thumb76_76.png",
"deadline": "2029-12-31T07:00:00Z",
"totalTeams": 0,
"totalKernels": 0,
"rewardQuantity": null,
"rewardTypeName": "Knowledge",
"organizationName": "ImageNet",
"organizationUrl": "image-net",
"hostSegment": "research",
"isLimited": false,
"isPrivate": false,
"isInClass": false,
"userHasEntered": false,
"rewardDisplay": "Knowledge"
},
{
"competitionId": 6796,
"competitionTitle": "ImageNet Object Detection Challenge",
"competitionDescription": "Identify and label everyday objects in images",
"competitionUrl": "/c/imagenet-object-detection-challenge",
"thumbnailImageUrl": "https://kaggle2.blob.core.windows.net/competitions/kaggle/6796/logos/thumb76_76.png",
"deadline": "2029-12-31T07:00:00Z",
"totalTeams": 0,
"totalKernels": 0,
"rewardQuantity": null,
"rewardTypeName": "Knowledge",
"organizationName": "ImageNet",
"organizationUrl": "image-net",
"hostSegment": "research",
"isLimited": false,
"isPrivate": false,
"isInClass": false,
"userHasEntered": false,
"rewardDisplay": "Knowledge"
},
{
"competitionId": 6800,
"competitionTitle": "ImageNet Object Detection from Video Challenge",
"competitionDescription": "Identify and label ordinary objects in videos",
"competitionUrl": "/c/imagenet-object-detection-from-video-challenge",
"thumbnailImageUrl": "https://kaggle2.blob.core.windows.net/competitions/kaggle/6800/logos/thumb76_76.png",
"deadline": "2029-12-31T07:00:00Z",
"totalTeams": 0,
"totalKernels": 0,
"rewardQuantity": null,
"rewardTypeName": "Knowledge",
"organizationName": "ImageNet",
"organizationUrl": "image-net",
"hostSegment": "research",
"isLimited": false,
"isPrivate": false,
"isInClass": false,
"userHasEntered": false,
"rewardDisplay": "Knowledge"
},
{
"competitionId": 6960,
"competitionTitle": "New York City Taxi Trip Duration",
"competitionDescription": "Share code and data to improve ride time predictions",
"competitionUrl": "/c/nyc-taxi-trip-duration",
"thumbnailImageUrl": "https://kaggle2.blob.core.windows.net/competitions/kaggle/6960/logos/thumb76_76.png",
"deadline": "2017-09-15T23:59:00Z",
"totalTeams": 546,
"totalKernels": 0,
"rewardQuantity": 30000,
"rewardTypeName": "USD",
"organizationName": "Kaggle",
"organizationUrl": "kaggle",
"hostSegment": "playground",
"isLimited": false,
"isPrivate": false,
"isInClass": false,
"userHasEntered": false,
"rewardDisplay": "$30,000"
},
{
"competitionId": 6469,
"competitionTitle": "Invasive Species Monitoring",
"competitionDescription": "Identify images of invasive hydrangea",
"competitionUrl": "/c/invasive-species-monitoring",
"thumbnailImageUrl": "https://kaggle2.blob.core.windows.net/competitions/kaggle/6469/logos/thumb76_76.png",
"deadline": "2017-08-15T23:59:00Z",
"totalTeams": 465,
"totalKernels": 0,
"rewardQuantity": null,
"rewardTypeName": "Knowledge",
"organizationName": null,
"organizationUrl": null,
"hostSegment": "playground",
"isLimited": false,
"isPrivate": false,
"isInClass": false,
"userHasEntered": false,
"rewardDisplay": "Knowledge"
},
{
"competitionId": 3136,
"competitionTitle": "Titanic: Machine Learning from Disaster",
"competitionDescription": "Start here! Predict survival on the Titanic and get familiar with ML basics",
"competitionUrl": "/c/titanic",
"thumbnailImageUrl": "https://kaggle2.blob.core.windows.net/competitions/kaggle/3136/logos/thumb76_76.png",
"deadline": "2020-04-07T00:00:00Z",
"totalTeams": 7367,
"totalKernels": 0,
"rewardQuantity": 0,
"rewardTypeName": "Knowledge",
"organizationName": "Kaggle",
"organizationUrl": "kaggle",
"hostSegment": "getting started",
"isLimited": false,
"isPrivate": false,
"isInClass": false,
"userHasEntered": false,
"rewardDisplay": "Knowledge"
},
{
"competitionId": 5407,
"competitionTitle": "House Prices: Advanced Regression Techniques",
"competitionDescription": "Predict sales prices and practice feature engineering, RFs, and gradient boosting",
"competitionUrl": "/c/house-prices-advanced-regression-techniques",
"thumbnailImageUrl": "https://kaggle2.blob.core.windows.net/competitions/kaggle/5407/logos/thumb76_76.png",
"deadline": "2020-03-01T23:59:00Z",
"totalTeams": 1613,
"totalKernels": 0,
"rewardQuantity": 0,
"rewardTypeName": "Knowledge",
"organizationName": "Kaggle",
"organizationUrl": "kaggle",
"hostSegment": "getting started",
"isLimited": false,
"isPrivate": false,
"isInClass": false,
"userHasEntered": false,
"rewardDisplay": "Knowledge"
},
{
"competitionId": 3004,
"competitionTitle": "Digit Recognizer",
"competitionDescription": "Learn computer vision fundamentals with the famous MNIST data",
"competitionUrl": "/c/digit-recognizer",
"thumbnailImageUrl": "https://kaggle2.blob.core.windows.net/competitions/kaggle/3004/logos/thumb76_76.png",
"deadline": "2020-01-07T00:00:00Z",
"totalTeams": 1731,
"totalKernels": 0,
"rewardQuantity": 0,
"rewardTypeName": "Knowledge",
"organizationName": "Kaggle",
"organizationUrl": "kaggle",
"hostSegment": "getting started",
"isLimited": false,
"isPrivate": false,
"isInClass": false,
"userHasEntered": false,
"rewardDisplay": "Knowledge"
}
],
"values": {
"page": 1,
"group": "active",
"sortBy": "prize",
"segment": null,
"search": ""
},
"banner": {
"bannerName": "competitions",
"isVisible": true,
"dismissBannerUrl": "/account/dismiss-banner?bannerName=competitions"
},
"canSeeNewWorldInClassCreation": false
}
```

ecosyste.ms

Data

Tools

Indexes

Applications

Experiments

Awesome

https://github.com/sdq/kaggle-crawler

Awesome Lists containing this project

README