Implement distribution of worker threads across multiple instances of the servers

.gitignore

@@ -45,3 +45,4 @@ data
 *.swp
 /.vscode/
 /bin/
+*.bkpignore

config/defaults/server.cfg

@@ -3,5 +3,7 @@
     "Port": 8000,
     "ProjectPath" : "./data/ProjectFiles/",
     "LogLevel": "Warn",
-    "TokenValidity": "1h"
+    "TokenValidity": "1h",
+    "RSAPrivateKeyLocation": "",
+    "RSAPrivateKeyPassword": ""
 }

modules/config/authconfig.go

@@ -0,0 +1,64 @@
+package config
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/pem"
+	"io/ioutil"
+
+	"github.com/CodeCollaborate/Server/utils"
+)
+
+func rsaConfigSetup(rsaPrivateKeyLocation, rsaPrivateKeyPassword string) (*rsa.PrivateKey, error) {
+	if rsaPrivateKeyLocation == "" {
+		utils.LogWarn("No RSA Key given, generating temp one", nil)
+		return GenRSA(4096)
+	}
+
+	priv, err := ioutil.ReadFile(rsaPrivateKeyLocation)
+	if err != nil {
+		utils.LogWarn("No RSA private key found, generating temp one", nil)
+		return GenRSA(4096)
+	}
+
+	privPem, _ := pem.Decode(priv)
+	var pemBytes []byte
+
+	if privPem.Type != "RSA PRIVATE KEY" {
+		utils.LogWarn("RSA private key is of the wrong type", utils.LogFields{
+			"Pem Type": privPem.Type,
+		})
+	}
+
+	if rsaPrivateKeyPassword != "" {
+		pemBytes, err = x509.DecryptPEMBlock(privPem, []byte(rsaPrivateKeyPassword))
+	} else {
+		pemBytes = privPem.Bytes
+	}
+
+	var parsedKey interface{}
+	if parsedKey, err = x509.ParsePKCS1PrivateKey(pemBytes); err != nil {
+		if parsedKey, err = x509.ParsePKCS8PrivateKey(pemBytes); err != nil { // note this returns type `interface{}`
+			utils.LogError("Unable to parse RSA private key, generating a temp one", err, utils.LogFields{})
+			return GenRSA(4096)
+		}
+	}
+
+	var privateKey *rsa.PrivateKey
+	var ok bool
+	if privateKey, ok = parsedKey.(*rsa.PrivateKey); !ok {
+		utils.LogError("Unable to parse RSA key, generating a temp one", err, utils.LogFields{})
+		return GenRSA(4096)
+	}
+
+	utils.LogInfo("Loaded RSA key from file", utils.LogFields{})
+	return privateKey, nil
+}
+
+// GenRSA returns a new RSA key of bits length
+func GenRSA(bits int) (*rsa.PrivateKey, error) {
+	key, err := rsa.GenerateKey(rand.Reader, bits)
+	utils.LogFatal("Failed to generate signing key", err, nil)
+	return key, err
+}

modules/config/authconfig_test.go

@@ -0,0 +1,18 @@
+package config
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAuthConfigSetup(t *testing.T) {
+	SetConfigDir("../../config")
+	err := LoadConfig()
+	assert.NoError(t, err, "error initializing config needed for password")
+
+	key, err := rsaConfigSetup("../../config/id_rsa", config.ServerConfig.RSAPrivateKeyPassword)
+	assert.Nil(t, err, "error loading rsa key")
+	assert.NotNil(t, key, "key was nil")
+	assert.NoError(t, key.Validate(), "key could not be validated")
+}

modules/config/manager.go

@@ -6,9 +6,9 @@ import (
 	"path/filepath"
 	"time"
 
-	"github.com/CodeCollaborate/Server/utils"
 	log "github.com/Sirupsen/logrus"
-	"github.com/kr/pretty"
+
+	"github.com/CodeCollaborate/Server/utils"
 )
 
 /**
@@ -35,7 +35,8 @@ func LoadConfig() error {
 
 	if err == nil {
 		utils.LogInfo("Loaded Configuration", utils.LogFields{
-			"ServerConfig": pretty.Sprint(config.ServerConfig),
+		//"ServerConfig": pretty.Sprint(config.ServerConfig),
+		// TODO: remove secret fields from config and then print again
 		})
 		setLogLevel()
 	}
@@ -69,6 +70,10 @@ func setLogLevel() {
 	}
 }
 
+func init() {
+	log.SetFormatter(&log.TextFormatter{DisableColors: true})
+}
+
 // EnableLoggingToFile redirects logger output to a logfile in the config's LogDir.
 // A new logfile will be created each time this method is called.
 func EnableLoggingToFile(logDir string) {
@@ -77,6 +82,7 @@ func EnableLoggingToFile(logDir string) {
 		logFile := filepath.Join(logDir, fmt.Sprintf("%d.%02d.%02d.%02d.%02d.log", time.Now().Year(), time.Now().Month(), time.Now().Day(), time.Now().Hour(), time.Now().Minute()))
 
 		log.Infof("Logging to %s", logFile)
+		log.SetFormatter(&log.JSONFormatter{})
 		f, err := os.OpenFile(logFile, os.O_WRONLY|os.O_CREATE, 0755)
 		if err != nil {
 			log.Error("Failed to setup logging to file")

modules/config/manager_test.go

@@ -4,14 +4,17 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
-	"reflect"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestGetConfig(t *testing.T) {
 	tmpDir := createTmpDir(t, ".", "test-config-files")
 	defer os.RemoveAll(tmpDir)
 
+	// ensure configDir is set to the default. other tests could have run before and set it elsewhere
+	SetConfigDir("./config")
 	err := LoadConfig()
 	if err == nil {
 		t.Fatal("Config dir not set yet; ./config does not exist. Should have failed.")
@@ -42,10 +45,16 @@ func TestGetConfig(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	assert.Nil(t, data.ServerConfig.rsaKey, "Ensure rsaKey lazily loaded")
+	privateKey := data.ServerConfig.RSAKey()
+	assert.NotNil(t, data.ServerConfig.rsaKey, "Ensure rsaKey lazily loaded")
+	assert.ObjectsAreEqual(privateKey, data.ServerConfig.rsaKey)
+
 	expected := &Config{
 		ServerConfig: ServerCfg{
-			Name: "CodeCollaborate",
-			Port: 80,
+			Name:   "CodeCollaborate",
+			Port:   80,
+			rsaKey: privateKey, // cheating
 		},
 		ConnectionConfig: ConnCfgMap{
 			"MySQL": ConnCfg{
@@ -63,7 +72,5 @@ func TestGetConfig(t *testing.T) {
 		},
 	}
 
-	if !reflect.DeepEqual(data, expected) {
-		t.Fatalf("Parsed data incorrect. Expected: \n%v\n Actual: \n%v\n", data, expected)
-	}
+	assert.ObjectsAreEqualValues(expected, data)
 }

modules/config/models.go

@@ -1,6 +1,11 @@
 package config
 
-import "time"
+import (
+	"crypto/rsa"
+	"time"
+
+	"github.com/CodeCollaborate/Server/utils"
+)
 
 /**
  * Models for the configuration CodeCollaborate Server.
@@ -18,14 +23,21 @@ type Config struct {
 // ServerCfg contains various config items that pertain to the server
 type ServerCfg struct {
 	Name            string
+	Host            string
 	Port            uint16
 	ProjectPath     string
 	DisableAuth     bool
+	UseTLS          bool
 	LogLevel        string
 	TokenValidity   string
 	MinBufferLength int
 	MaxBufferLength int
 
+	// RSA key
+	RSAPrivateKeyLocation string
+	RSAPrivateKeyPassword string
+	rsaKey                *rsa.PrivateKey
+
 	// Parsed validity
 	tokenValidityDuration time.Duration
 }
@@ -41,6 +53,20 @@ func (cfg ServerCfg) TokenValidityDuration() (time.Duration, error) {
 	return cfg.tokenValidityDuration, err
 }
 
+// RSAKey returns the RSA key the server should use for signing tokens
+func (cfg *ServerCfg) RSAKey() *rsa.PrivateKey {
+	if cfg.rsaKey != nil {
+		return cfg.rsaKey
+	}
+
+	var err error
+	cfg.rsaKey, err = rsaConfigSetup(config.ServerConfig.RSAPrivateKeyLocation, config.ServerConfig.RSAPrivateKeyPassword)
+	if err != nil {
+		utils.LogFatal("Unable to load/generate RSA key", err, utils.LogFields{})
+	}
+	return cfg.rsaKey
+}
+
 // ConnCfg represents the information required to make a connection
 type ConnCfg struct {
 	Host       string

modules/datahandling/authentication.go

@@ -1,13 +1,16 @@
 package datahandling
 
 import (
+	"crypto/rsa"
 	"errors"
 	"fmt"
 	"strings"
 	"time"
 
-	"github.com/CodeCollaborate/Server/modules/config"
 	"github.com/dgrijalva/jwt-go"
+
+	"github.com/CodeCollaborate/Server/modules/config"
+	"github.com/CodeCollaborate/Server/utils"
 )
 
 type tokenPayload struct {
@@ -16,6 +19,22 @@ type tokenPayload struct {
 	Validity     int64
 }
 
+var rsaKey *rsa.PrivateKey
+
+func getRsaKey() *rsa.PrivateKey {
+	if rsaKey != nil {
+		return rsaKey
+	}
+
+	cfg := config.GetConfig()
+	if cfg == nil {
+		utils.LogFatal("Failed to load RSA key from config", errors.New("config not initialized"), utils.LogFields{})
+	}
+
+	rsaKey = cfg.ServerConfig.RSAKey()
+	return rsaKey
+}
+
 // Valid is the (unused) method to determine if the token is valid. however, since we need to have a reference
 // to the abstract request, we cannot do validation here. Token validation has been shifted to the authenticate
 // method. This is here for conformance to the token.Claims interface.
@@ -26,10 +45,10 @@ func (tokenPayload) Valid() error {
 func authenticate(abs abstractRequest) error {
 	token, err := jwt.ParseWithClaims(abs.SenderToken, &tokenPayload{}, func(token *jwt.Token) (interface{}, error) {
 		// Don't forget to validate the alg is what you expect:
-		if _, ok := token.Method.(*jwt.SigningMethodECDSA); !ok {
+		if _, ok := token.Method.(*jwt.SigningMethodRSA); !ok {
 			return nil, fmt.Errorf("ParseWithClaims - Unexpected signing method: %v", token.Header["alg"])
 		}
-		return &privKey.PublicKey, nil
+		return &getRsaKey().PublicKey, nil
 	})
 	if err != nil {
 		return fmt.Errorf("authenticate - failed to parse token: %s", err)
@@ -40,7 +59,8 @@ func authenticate(abs abstractRequest) error {
 		if !strings.EqualFold(claims.Username, abs.SenderID) {
 			return errors.New("authenticate - senderID did not match token username")
 		}
-		if time.Unix(claims.CreationTime, 0).After(time.Now()) {
+		// check if token is valid yet, but now with some wiggle room
+		if time.Unix(claims.CreationTime, 0).Add(-1 * time.Minute).After(time.Now()) {
 			return errors.New("authenticate - token not valid yet")
 		}
 		if !time.Unix(claims.Validity, 0).After(time.Now()) {
@@ -58,11 +78,11 @@ func newAuthToken(username string) (string, error) {
 		return "", err
 	}
 
-	token := jwt.NewWithClaims(jwt.SigningMethodES256, tokenPayload{
+	token := jwt.NewWithClaims(jwt.SigningMethodRS512, tokenPayload{
 		Username:     username,
 		CreationTime: time.Now().Unix(),
 		Validity:     time.Now().Add(tokenValidityDuration).Unix(),
 	})
 
-	return token.SignedString(privKey)
+	return token.SignedString(getRsaKey())
 }