From 2d0d821b3d4779323ce11ebd38d33813c7ddf042 Mon Sep 17 00:00:00 2001 From: Antoine Rousseau Date: Mon, 26 Aug 2019 13:09:52 +0200 Subject: [PATCH] handle videos --- README.md | 5 ++- google.js | 58 +++++++++++++------------ start.js | 128 ++++++++++++++++++++++++++++++++++++++---------------- 3 files changed, 125 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index de8a8f7..2c22129 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,10 @@ Then create a `.env` file based on `example.env` (`cp {example,}.env`), and edit ./start.js +By default, it will process all photos and videos. If you want only photos or only videos, use the `MEDIA` environment variable. Set it to `photos` or `videos`, e.g.: + + MEDIA=photos ./start.js + ## Daemonize You can use [PM2](https://github.com/Unitech/pm2) @@ -28,5 +32,4 @@ You can use [PM2](https://github.com/Unitech/pm2) ## Limitations -- This script does not handle videos ([yet](https://github.com/antoinerousseau/flickr2google/pull/1)) - If your [Google storage](https://drive.google.com/settings/storage) is limited and you hit the limit, the Google API will return a "Bad Request". You must then either buy more storage, or go to your [Google Photos settings](https://photos.google.com/settings), choose "High Quality" and click "Recover storage". This will convert your uploads to [16 Megapixels compressed photos](https://support.google.com/photos/answer/6220791), which the API cannot do on the fly. Also, you can only convert once per day. diff --git a/google.js b/google.js index cb8f25e..7fe8bca 100644 --- a/google.js +++ b/google.js @@ -3,7 +3,7 @@ const request = require("request") const HttpError = require("standard-http-error") const { GOOGLE_FILE_PATH, CALLBACK_URL, GOOGLE_API_ENDPOINT } = require("./constants") -const { log, readJson, writeJson, prompt } = require("./utils") +const { log, logError, readJson, writeJson, prompt } = require("./utils") module.exports = async () => { let googleTokens @@ -53,35 +53,39 @@ module.exports = async () => { const stream = async (url, filename) => { await refreshTokenIfNeeded() return new Promise((resolve, reject) => { - request - .get(url) - // .on("response", (response) => { - // log("downloaded", response.statusCode, response.statusMessage, response.headers) - // }) + const req = request.get(url) + req .on("error", (error) => { reject(error) }) - .pipe( - request.post( - { - url: GOOGLE_API_ENDPOINT + "uploads", - headers: { - // "Content-type": "application/octet-stream", - // "Content-length": set by stream - "X-Goog-Upload-File-Name": filename, - "X-Goog-Upload-Protocol": "raw", - Authorization: `Bearer ${googleTokens.access_token}`, - }, - }, - (error, response, body) => { - if (error) { - reject(error) - } else { - resolve(body) - } - } - ) - ) + .on("response", (response) => { + if (response.statusCode >= 400) { + logError("Could not download", url) + reject(new HttpError(response.statusCode, response.statusMessage)) + } else { + req.pipe( + request.post( + { + url: GOOGLE_API_ENDPOINT + "uploads", + headers: { + "Content-type": "application/octet-stream", + // "Content-length": set by stream + "X-Goog-Upload-File-Name": filename, + "X-Goog-Upload-Protocol": "raw", + Authorization: `Bearer ${googleTokens.access_token}`, + }, + }, + (error, response, body) => { + if (error) { + reject(error) + } else { + resolve(body) + } + } + ) + ) + } + }) }) } diff --git a/start.js b/start.js index fb92132..4f9f747 100755 --- a/start.js +++ b/start.js @@ -6,14 +6,16 @@ const googleConnect = require("./google") const { ALBUMS_PATH } = require("./constants") const { log, logError, readJson, writeJson, fileExists, mkdir } = require("./utils") +const media = process.env.MEDIA || "all" // "photos" or "videos" +const extras = "url_o,media,path_alias,original_format" +const per_page = 500 // max + const getAlbumPath = (id) => `${ALBUMS_PATH}/${id}.json` if (!fileExists(ALBUMS_PATH)) { mkdir(ALBUMS_PATH) } -const per_page = 500 // max - const main = async () => { const { flickr, user_id } = await flickrConnect() const { stream, post } = await googleConnect() @@ -31,45 +33,64 @@ const main = async () => { id: "NotInSet", }) - const albums_cache = {} + const memory = {} photosets.forEach((set) => { const path = getAlbumPath(set.id) if (fileExists(path)) { - albums_cache[set.id] = readJson(path) - if (set.photos && albums_cache[set.id].num_photos !== set.photos) { + memory[set.id] = readJson(path) + if (set.photos != null && memory[set.id].num_photos !== set.photos) { // number of photos has changed since last time, update: - albums_cache[set.id].num_photos = set.photos - writeJson(path, albums_cache[set.id]) + memory[set.id].num_photos = set.photos + writeJson(path, memory[set.id]) + } + if (set.videos != null && memory[set.id].num_videos !== set.videos) { + // number of videos has changed since last time, update: + memory[set.id].num_videos = set.videos + writeJson(path, memory[set.id]) } } else { - albums_cache[set.id] = { - title: set.title && set.title._content, + memory[set.id] = { + title: set.id === "NotInSet" ? "Not in a set" : set.title._content, flickr_set: set.id, google_album: null, num_photos: set.photos, + num_videos: set.videos, + total: set.photos + set.videos, + uploaded_photos: 0, + uploaded_videos: 0, done: [], } - writeJson(path, albums_cache[set.id]) + writeJson(path, memory[set.id]) } }) for (let i = 0; i < photosets.length; i++) { const photoset_id = photosets[i].id const path = getAlbumPath(photoset_id) - const data = albums_cache[photoset_id] - if (data.done.length === data.num_photos) { + const data = memory[photoset_id] + + if (media === "photos" && data.num_photos === data.uploaded_photos) { + continue + } + if (media === "videos" && data.num_videos === data.uploaded_videos) { + continue + } + if (media === "all" && data.num_photos + data.num_videos === data.uploaded_photos + data.uploaded_videos) { continue } - // FOR EACH PHOTOSET, RETRIEVE PHOTOS + const total_photos = data.num_photos == null ? "?" : data.num_photos + const total_videos = data.num_videos == null ? "?" : data.num_videos log( - `Processing "${data.title || photoset_id}" set ${i + 1}/${photosets.length};`, + `Fetching ${media} in "${data.title}" set ${i + 1}/${photosets.length};`, `Flickr id: ${photoset_id};`, - `Total: ${data.num_photos || "?"} photos` + `Total: ${total_photos} photos & ${total_videos} videos` ) + // FOR EACH PHOTOSET, RETRIEVE PHOTOS + let photoset let page = 0 do { @@ -77,16 +98,28 @@ const main = async () => { if (photoset_id === "NotInSet") { const { body } = await flickr.photos.getNotInSet({ // https://www.flickr.com/services/api/flickr.photos.getNotInSet.html - media: "photos", - extras: "url_o", - page, + media, + extras, per_page, + page, }) photoset = body.photos - photoset.title = "Photos not in a set" - if (!data.num_photos) { - data.num_photos = Number(photoset.total) + const totalNotInSet = Number(photoset.total) + + if (!totalNotInSet) { + continue + } + if (media === "photos" && data.num_photos == null) { + data.num_photos = totalNotInSet + writeJson(path, data) + } + if (media === "videos" && data.num_videos == null) { + data.num_videos = totalNotInSet + writeJson(path, data) + } + if (media === "all" && data.total == null) { + data.total = totalNotInSet writeJson(path, data) } } else { @@ -94,22 +127,25 @@ const main = async () => { // https://www.flickr.com/services/api/flickr.photosets.getPhotos.html photoset_id, user_id, - media: "photos", - extras: "url_o", - page, + media, + extras, per_page, + page, }) photoset = body.photoset } - const { title, photo: photos, pages } = photoset + const { photo: items, pages } = photoset - log(`Processing page ${page}/${pages} (${photos.length} photos);`, `Done ${data.done.length}`) + log( + `Processing page ${page}/${pages} (${items.length} ${media === "all" ? "items" : media});`, + media === "all" ? `Done ${data.done.length}` : "" + ) if (!data.google_album) { const albumRequest = { album: { - title, + title: data.title, }, } const { json: album } = await post("albums", albumRequest) @@ -118,22 +154,37 @@ const main = async () => { writeJson(path, data) } - for (let j = 0; j < photos.length; j++) { - const photo = photos[j] - if (data.done.includes(photo.id)) { + for (let j = 0; j < items.length; j++) { + const item = items[j] + if (data.done.includes(item.id)) { continue } + let url + if (item.media === "video") { + url = `https://www.flickr.com/photos/${item.pathalias}/${item.id}/play/orig/${item.originalsecret}/` + // TODO: does not work for some videos + // https://www.flickr.com/groups/51035612836@N01/discuss/72157621698855558/ + } else { + url = item.url_o + } + // FOR EACH PHOTO, UPLOAD TO GOOGLE PHOTOS - const uploadToken = await stream(photo.url_o, `flickr_${photo.id}.jpg`) + let uploadToken + try { + uploadToken = await stream(url, `flickr_${item.id}.${item.originalformat}`) + } catch (err) { + logError(err) + continue + } - const media = { + const mediaItem = { // https://developers.google.com/photos/library/reference/rest/v1/mediaItems/batchCreate albumId: data.google_album, newMediaItems: [ { - description: photo.title, + description: item.title, simpleMediaItem: { uploadToken, }, @@ -143,18 +194,19 @@ const main = async () => { const { json: { newMediaItemResults: results }, status, - } = await post("mediaItems:batchCreate", media) + } = await post("mediaItems:batchCreate", mediaItem) if (status === 200) { - if (results.length === 1 && results[0].mediaItem) { - data.done.push(photo.id) + if (results.length === 1 && results[0].mediaItem && !results[0].status.code) { + data.done.push(item.id) + data[`uploaded_${item.media}s`]++ writeJson(path, data) log("Created media item @", results[0].mediaItem.productUrl) } else { - logError("Media Item creation status 200 OK but wrong response:", results) + logError("Media Item creation status 200 OK but wrong response:", results[0].status) } } else { - logError("Could not create media item", results[0]) + logError("Could not create media item", results[0].status) } } } while (page < photoset.pages)