uploading project to git

.gitignore

@@ -0,0 +1 @@
+node_modules/

README.md

@@ -0,0 +1,66 @@
+# MERN Stack Boilerplate
+
+## Setup
+
+#### Node
+
+For Linux:
+```
+curl -sL https://deb.nodesource.com/setup_13.x | sudo -E bash -
+sudo apt-get install -y nodejs
+```
+
+For Mac:
+```
+brew install node
+```
+
+#### MongoDB
+
+Install the community edition [here](https://docs.mongodb.com/manual/installation/#mongodb-community-edition-installation-tutorials).
+
+#### React
+
+```
+npm install -g create-react-app
+```
+
+To create a new React app:
+```
+create-react-app name_of_app
+```
+
+To run the app, cd into the directory and do:
+```
+npm start
+```
+
+## Running the boilerplate
+
+Run Mongo daemon:
+```
+sudo mongod
+```
+Mongo will be running on port 27017.
+
+To create a database:
+```
+mongo
+``` 
+This will open the mongo shell. Type in ```use users``` to create a new database called users.
+
+Run Express:
+```
+cd backend/
+npm install
+npm start
+```
+
+Run React:
+```
+cd frontend
+npm install/
+npm start
+```
+Navigate to localhost:3000/ in your browser.
+

backend/config/passport.js

@@ -0,0 +1,26 @@
+var passport = require("passport");
+var GoogleStrategy = require("passport-google-oauth").OAuth2Strategy;
+passport.serializeUser(function(user, done) {
+ done(null, user);
+});
+passport.deserializeUser(function(user, done) {
+ done(null, user);
+});
+passport.use(
+ new GoogleStrategy(
+  {
+   clientID: "122868526299-9lvnl131gjii2c1ri99neo2j4rq6tvip.apps.googleusercontent.com",
+   clientSecret: "yHsATwJ7OyeHdss6YghHjN6s",
+   callbackURL: "http://localhost:4000/auth/google/callback"
+  },
+  function(accessToken, refreshToken, profile, done) {
+   var userData = {
+    email: profile.emails[0].value,
+    name: profile.displayName,
+    token: accessToken
+   };
+   done(null, userData);
+  }
+ )
+);
+

backend/crawl.sh

@@ -0,0 +1,7 @@
+# !/bib/bash
+
+cd data_crawling
+rm schools.json 2> /dev/null
+scrapy crawl prototype -o schools.json --nolog
+cd ..
+cat data_crawling/schools.json

backend/crawl_startups.sh

@@ -0,0 +1,11 @@
+# !/bib/bash
+cd data_crawling
+rm items.json 2> /dev/null
+if test -z "$2"
+then
+    scrapy crawl startupprototype -o items.json -t json -a domain="https://startuptracker.io/discover?filters[0][bm][o]=inc&filters[0][bm][q]=$1" --nolog >/dev/null
+else
+    scrapy crawl startupprototype -o items.json -t json -a domain="https://startuptracker.io/discover?filters[0][bm][o]=inc&filters[0][bm][q]=$1&filters[1][cc][q]=$2" --nolog >/dev/null
+fi
+cd ..
+cat data_crawling/items.json

backend/crawl_startups_ind.sh

@@ -0,0 +1,6 @@
+# !/bib/bash
+cd data_crawling
+rm items_ind.json 2> /dev/null
+scrapy crawl indstupprototype -o items_ind.json -t json -a url=$1
+cd ..
+cat data_crawling/items_ind.json

backend/data_crawling/README.md

@@ -0,0 +1,23 @@
+# DASS25
+
+Team 25
+
+## Protoype for the phrase "school management system"
+
+### System requirements
+
+* `python3`
+* `scrapy` module
+
+### Running
+
+* Go to 'prototype' folder
+* Run the command 
+
+    `scrapy crawl prototype`
+* To store the result in schools.json run the command 
+
+    `scrapy crawl prototype -o schools.json`
+* To view the result from file use the command
+
+    `python3 -m json.tool schools.json`

backend/data_crawling/items.json

@@ -0,0 +1,8 @@
+[
+{"title": null, "description": null, "domains": [], "url": null, "image": null},
+{"title": null, "description": null, "domains": [], "url": null, "image": null},
+{"title": null, "description": null, "domains": [], "url": null, "image": null},
+{"title": null, "description": null, "domains": [], "url": null, "image": null},
+{"title": null, "description": null, "domains": [], "url": null, "image": null},
+{"title": null, "description": null, "domains": [], "url": null, "image": null}
+]

backend/data_crawling/prototpye/items.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+
+# Define here the models for your scraped items
+#
+# See documentation in:
+# https://doc.scrapy.org/en/latest/topics/items.html
+
+# import scrapy
+
+
+# class PrototpyeItem(scrapy.Item):
+#     # define the fields for your item here like:
+#     # name = scrapy.Field()
+#     pass
+from scrapy.item import Item, Field
+
+
+class StackItem(Item):
+    title = Field()
+    address = Field()

backend/data_crawling/prototpye/middlewares.py

@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+
+# Define here the models for your spider middleware
+#
+# See documentation in:
+# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
+
+from scrapy import signals
+
+
+class PrototpyeSpiderMiddleware(object):
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the spider middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_spider_input(self, response, spider):
+        # Called for each response that goes through the spider
+        # middleware and into the spider.
+
+        # Should return None or raise an exception.
+        return None
+
+    def process_spider_output(self, response, result, spider):
+        # Called with the results returned from the Spider, after
+        # it has processed the response.
+
+        # Must return an iterable of Request, dict or Item objects.
+        for i in result:
+            yield i
+
+    def process_spider_exception(self, response, exception, spider):
+        # Called when a spider or process_spider_input() method
+        # (from other spider middleware) raises an exception.
+
+        # Should return either None or an iterable of Response, dict
+        # or Item objects.
+        pass
+
+    def process_start_requests(self, start_requests, spider):
+        # Called with the start requests of the spider, and works
+        # similarly to the process_spider_output() method, except
+        # that it doesn’t have a response associated.
+
+        # Must return only requests (not items).
+        for r in start_requests:
+            yield r
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)
+
+
+class PrototpyeDownloaderMiddleware(object):
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the downloader middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_request(self, request, spider):
+        # Called for each request that goes through the downloader
+        # middleware.
+
+        # Must either:
+        # - return None: continue processing this request
+        # - or return a Response object
+        # - or return a Request object
+        # - or raise IgnoreRequest: process_exception() methods of
+        #   installed downloader middleware will be called
+        return None
+
+    def process_response(self, request, response, spider):
+        # Called with the response returned from the downloader.
+
+        # Must either;
+        # - return a Response object
+        # - return a Request object
+        # - or raise IgnoreRequest
+        return response
+
+    def process_exception(self, request, exception, spider):
+        # Called when a download handler or a process_request()
+        # (from other downloader middleware) raises an exception.
+
+        # Must either:
+        # - return None: continue processing this exception
+        # - return a Response object: stops process_exception() chain
+        # - return a Request object: stops process_exception() chain
+        pass
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)

backend/data_crawling/prototpye/pipelines.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+
+# Define your item pipelines here
+#
+# Don't forget to add your pipeline to the ITEM_PIPELINES setting
+# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
+
+
+# class PrototpyePipeline(object):
+#     def process_item(self, item, spider):
+#         return item
+import pymongo
+
+from scrapy.conf import settings
+from scrapy.exceptions import DropItem
+from scrapy import log
+
+
+class PrototpyePipeline(object):
+
+    def __init__(self):
+        connection = pymongo.MongoClient(
+            settings['MONGODB_SERVER'],
+            settings['MONGODB_PORT']
+        )
+        db = connection[settings['MONGODB_DB']]
+        self.collection = db[settings['MONGODB_COLLECTION']]
+
+    def process_item(self, item, spider):
+        valid = True
+        for data in item:
+            if not data:
+                valid = False
+                raise DropItem("Missing {0}!".format(data))
+        if valid:
+            self.collection.insert(dict(item))
+            log.msg("Question added to MongoDB database!",
+                    level=log.DEBUG, spider=spider)
+        return item