Local dev + full d3 v7 / hedotools v7 / d3-shifterator v7 migration

.gitignore

@@ -27,3 +27,6 @@ scratch
 # data for the database
 scripts/data/
 pyenv38/
+
+# stray sqlite fallback (real DB is postgres)
+db.sqlite3

hedonometer/migrations/0002_auto_20200413_1227.py

@@ -16,7 +16,12 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='embeddable',
             name='author',
-            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='twython_django.TwitterProfile'),
+            # Originally a FK to twython_django.TwitterProfile; that app is no
+            # longer installed, this field is removed in migration 0010, and the
+            # Embeddable model is deleted in 0011 -- so the target is irrelevant
+            # to the final schema. Repointed to 'self' so a fresh `migrate`
+            # (new dev env / CI) can replay without the defunct twython-django.
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='hedonometer.Embeddable'),
         ),
         migrations.AddField(
             model_name='band',

hedonometer/static/hedonometer/js/books-on-load-harry.js

@@ -40,7 +40,7 @@ function initializePlot() {
     book = bookDecoder().cached;
     // console.log("not a classic");
     // hit the random api
-    d3.json("https://hedonometer.org/api/v1/gutenberg/?format=json&title__exact="+book,function(data) {
+    d3.json("https://hedonometer.org/api/v1/gutenberg/?format=json&title__exact="+book).then(function(data) {
 	var result = data.objects[0];
 	console.log(result);
 	lang = result.language;
@@ -71,15 +71,15 @@ function initializePlot() {
 function loadCsv() {
     var csvLoadsRemaining = 4;
     var bookfile = "https://hedonometer.org/data/bookdata/timeseries/"+bookref+".csv";
-    d3.text(bookfile, function (text) {
+    d3.text(bookfile).then(function (text) {
 	var tmpminwin = 10;
 	fulltimeseries = text.split(",").map(parseFloat);
 	begtimeseries = fulltimeseries.slice(0,tmpminwin/2+1);
 	endtimeseries = fulltimeseries.slice(fulltimeseries.length-1-tmpminwin/2,fulltimeseries.length);
 	timeseries = fulltimeseries.slice(tmpminwin/2,fulltimeseries.length-tmpminwin/2);
         if (!--csvLoadsRemaining) initializePlotPlot();
     });
-    d3.text("https://hedonometer.org/data/bookdata/labMT/labMTscores-"+lang+".csv", function (text) {
+    d3.text("https://hedonometer.org/data/bookdata/labMT/labMTscores-"+lang+".csv").then(function (text) {
         var tmp = text.split("\n");
         //console.log(tmp.length);
         //console.log(tmp[tmp.length-1]);
@@ -93,7 +93,7 @@ function loadCsv() {
 	hedotools.shifter._lens(lens);
         if (!--csvLoadsRemaining) initializePlotPlot();
     });
-    d3.text("https://hedonometer.org/data/bookdata/labMT/labMTwords-"+lang+".csv", function (text) {
+    d3.text("https://hedonometer.org/data/bookdata/labMT/labMTwords-"+lang+".csv").then(function (text) {
         var tmp = text.split("\n");
         var words = tmp;
         var len = words.length - 1;
@@ -105,7 +105,7 @@ function loadCsv() {
 	hedotools.shifter._words(words);
         if (!--csvLoadsRemaining) initializePlotPlot();
     });
-    d3.text("https://hedonometer.org/data/bookdata/labMT/labMTwordsEn-"+lang+".csv", function (text) {
+    d3.text("https://hedonometer.org/data/bookdata/labMT/labMTwordsEn-"+lang+".csv").then(function (text) {
         var tmp = text.split("\n");
         var words_en = tmp;
         var len = words_en.length - 1;
@@ -189,7 +189,7 @@ var loadwordshiftdata = function() {
     var spinner = new Spinner(opts).spin(target);
 
     var bookfile = "https://hedonometer.org/data/bookdata/word-vectors/"+bookref+".csv";
-    d3.text(bookfile, function (text) {
+    d3.text(bookfile).then(function (text) {
         tmp = text.split("\n");
 
         // build the full data, terrible
@@ -400,7 +400,7 @@ var substringMatcher = function(apik) {
         //     }
         // }
         // if (matches.length === 0) { matches.push({ value: "<i>book not indexed</i>" }); }
-	d3.json("https://hedonometer.org/api/v1/gutenberg/?format=json&"+apik.toLowerCase()+"__icontains="+q,function(data) {
+	d3.json("https://hedonometer.org/api/v1/gutenberg/?format=json&"+apik.toLowerCase()+"__icontains="+q).then(function(data) {
 	    var result = data.objects;
 	    // console.log(result);
 	    var newresult = [];
@@ -424,7 +424,7 @@ $(document).ready(function() {
 	// window.location.replace("/books.html?book=random");
 	// but here, it's easier to make that API hit now and send the page
 	// directly:
-	d3.json("https://hedonometer.org/api/v1/randombook/?format=json",function(data) {
+	d3.json("https://hedonometer.org/api/v1/randombook/?format=json").then(function(data) {
 	    var result = data.objects[0];
 	    window.location.replace("/books/"+result.title+"/");
 	})	

hedonometer/static/hedonometer/js/books-on-load.js

@@ -38,7 +38,7 @@ function initializePlot() {
     book = bookDecoder().cached;
     // hit the random api
     if (book === 'random') {
-	d3.json("https://hedonometer.org/api/v1/randombook/?format=json",function(data) {
+	d3.json("https://hedonometer.org/api/v1/randombook/?format=json").then(function(data) {
 	    var result = data.objects[0];
 	    // console.log(result);
 	    lang = result.language;
@@ -63,7 +63,7 @@ function initializePlot() {
 	})
     }
     else {
-	d3.json("https://hedonometer.org/api/v1/gutenberg/?format=json&title__exact="+book,function(data) {
+	d3.json("https://hedonometer.org/api/v1/gutenberg/?format=json&title__exact="+book).then(function(data) {
 	    var result = data.objects[0];
 	    console.log(result);
 	    lang = result.language;
@@ -91,7 +91,7 @@ function initializePlot() {
 function loadCsv() {
     var csvLoadsRemaining = 4;
     var bookfile = "https://hedonometer.org/data/bookdata/processed/"+book+".csv";
-    d3.text(bookfile, function (text) {
+    d3.text(bookfile).then(function (text) {
         tmp = text.split("\n");
         // kill extra rows
         var len = tmp.length - 1;
@@ -112,7 +112,7 @@ function loadCsv() {
         //console.log(d3.sum(allDataRaw[0]));
         if (!--csvLoadsRemaining) initializePlotPlot();
     });
-    d3.text("https://hedonometer.org/data/bookdata/labMT/labMTscores-"+lang+".csv", function (text) {
+    d3.text("https://hedonometer.org/data/bookdata/labMT/labMTscores-"+lang+".csv").then(function (text) {
         var tmp = text.split("\n");
         //console.log(tmp.length);
         //console.log(tmp[tmp.length-1]);
@@ -126,7 +126,7 @@ function loadCsv() {
 	hedotools.shifter._lens(lens);
         if (!--csvLoadsRemaining) initializePlotPlot();
     });
-    d3.text("https://hedonometer.org/data/bookdata/labMT/labMTwords-"+lang+".csv", function (text) {
+    d3.text("https://hedonometer.org/data/bookdata/labMT/labMTwords-"+lang+".csv").then(function (text) {
         var tmp = text.split("\n");
         words = tmp;
         var len = words.length - 1;
@@ -138,7 +138,7 @@ function loadCsv() {
 	hedotools.shifter._words(words);
         if (!--csvLoadsRemaining) initializePlotPlot();
     });
-    d3.text("https://hedonometer.org/data/bookdata/labMT/labMTwordsEn-"+lang+".csv", function (text) {
+    d3.text("https://hedonometer.org/data/bookdata/labMT/labMTwordsEn-"+lang+".csv").then(function (text) {
         var tmp = text.split("\n");
         words_en = tmp;
         var len = words_en.length - 1;
@@ -259,7 +259,7 @@ var substringMatcher = function(apik) {
         //     }
         // }
         // if (matches.length === 0) { matches.push({ value: "<i>book not indexed</i>" }); }
-	d3.json("https://hedonometer.org/api/v1/gutenberg/?format=json&"+apik.toLowerCase()+"__icontains="+q,function(data) {
+	d3.json("https://hedonometer.org/api/v1/gutenberg/?format=json&"+apik.toLowerCase()+"__icontains="+q).then(function(data) {
 	    var result = data.objects;
 	    // console.log(result);
 	    var newresult = [];

hedonometer/static/hedonometer/js/books/books-on-load-v3.js

@@ -30,7 +30,7 @@ function commaSeparateNumber(val){
 var my_shifter;
 
 function initializePlot() {
-    my_shifter = hedotools.shifter();
+    my_shifter = hedotools.shifter;
     loadCsv();
 }
 
@@ -44,7 +44,7 @@ function loadCsv() {
         --csvLoadsRemaining;
     }
     else {
-        d3.text("http://hedonometer.org/data/bookdata/labMT/labMTwordsEn-"+bookinfo.lang+".csv", function (text) {
+        d3.text("http://hedonometer.org/data/bookdata/labMT/labMTwordsEn-"+bookinfo.lang+".csv").then(function (text) {
             var tmp = text.split("\n");
             words_en = tmp;
             var len = words_en.length - 1;
@@ -57,16 +57,7 @@ function loadCsv() {
             if (!--csvLoadsRemaining) initializePlotPlot();
         });
     }
-    d3.text(bookfile, function (error,text) {
-        if (error) {
-            missing_file = true;
-            console.log("file is missing");
-            bookinfo.avhapps = "N/A";
-            bookinfo.len = "N/A";
-            cat_card(bookinfo);
-            d3.select("#booktitle").append("h4").html("** book file not available");
-            return;
-        }
+    d3.text(bookfile).then(function (text) {
         tmp = text.split("\n");
         // kill extra rows
         var len = tmp.length - 1;
@@ -86,8 +77,16 @@ function loadCsv() {
 	if (bookinfo.length < 10000) { alert("There are too few words in this book for the hedonometer to accurately generate a timeseries. Currently we need at least 10000 words, and this book has "+bookinfo.length+"."); }
         //console.log(d3.sum(allDataRaw[0]));
         if (!--csvLoadsRemaining) initializePlotPlot();
+    }).catch(function (error) {
+        missing_file = true;
+        console.log("file is missing");
+        bookinfo.avhapps = "N/A";
+        bookinfo.len = "N/A";
+        cat_card(bookinfo);
+        d3.select("#booktitle").append("h4").html("** book file not available");
+        return;
     });
-    d3.text("http://hedonometer.org/data/bookdata/labMT/labMTscores-"+bookinfo.lang+".csv", function (text) {
+    d3.text("http://hedonometer.org/data/bookdata/labMT/labMTscores-"+bookinfo.lang+".csv").then(function (text) {
         var tmp = text.split("\n");
         //console.log(tmp.length);
         //console.log(tmp[tmp.length-1]);
@@ -101,7 +100,7 @@ function loadCsv() {
 	my_shifter._lens(lens);
         if (!--csvLoadsRemaining) initializePlotPlot();
     });
-    d3.text("http://hedonometer.org/data/bookdata/labMT/labMTwords-"+bookinfo.lang+".csv", function (text) {
+    d3.text("http://hedonometer.org/data/bookdata/labMT/labMTwords-"+bookinfo.lang+".csv").then(function (text) {
         var tmp = text.split("\n");
         words = tmp;
         var len = words.length - 1;
@@ -251,7 +250,7 @@ var substringMatcher = function(apik1,apik2) {
         //     }
         // }
         // if (matches.length === 0) { matches.push({ value: "<i>book not indexed</i>" }); }
-	d3.json(query_string+q,function(data) {
+	d3.json(query_string+q).then(function(data) {
 	    var result = data.objects;
 	    console.log(result);
 	    var newresult = [];
@@ -282,7 +281,7 @@ $(document).ready(function() {
 	    window.location.replace("/books/v3/"+(Math.floor((Math.random() * 51249) + 1)).toFixed(0)+"/");
         }
         else {
-            d3.text("/static/hedonometer/gut_ids.txt", function (text) {
+            d3.text("/static/hedonometer/gut_ids.txt").then(function (text) {
                 var tmp = text.split("\n");
                 var rand_ind = Math.floor((Math.random() * tmp.length));
                 window.location.replace("/books/v3/"+tmp[rand_ind]+"/");