diff --git a/.gitignore b/.gitignore index 586f709..ba7eefc 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,4 @@ data *.swp /.vscode/ /bin/ +*.bkpignore diff --git a/config/defaults/server.cfg b/config/defaults/server.cfg index cde5ab4..5cfa159 100644 --- a/config/defaults/server.cfg +++ b/config/defaults/server.cfg @@ -3,5 +3,7 @@ "Port": 8000, "ProjectPath" : "./data/ProjectFiles/", "LogLevel": "Warn", - "TokenValidity": "1h" + "TokenValidity": "1h", + "RSAPrivateKeyLocation": "", + "RSAPrivateKeyPassword": "" } diff --git a/modules/config/authconfig.go b/modules/config/authconfig.go new file mode 100644 index 0000000..392970a --- /dev/null +++ b/modules/config/authconfig.go @@ -0,0 +1,64 @@ +package config + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "io/ioutil" + + "github.com/CodeCollaborate/Server/utils" +) + +func rsaConfigSetup(rsaPrivateKeyLocation, rsaPrivateKeyPassword string) (*rsa.PrivateKey, error) { + if rsaPrivateKeyLocation == "" { + utils.LogWarn("No RSA Key given, generating temp one", nil) + return GenRSA(4096) + } + + priv, err := ioutil.ReadFile(rsaPrivateKeyLocation) + if err != nil { + utils.LogWarn("No RSA private key found, generating temp one", nil) + return GenRSA(4096) + } + + privPem, _ := pem.Decode(priv) + var pemBytes []byte + + if privPem.Type != "RSA PRIVATE KEY" { + utils.LogWarn("RSA private key is of the wrong type", utils.LogFields{ + "Pem Type": privPem.Type, + }) + } + + if rsaPrivateKeyPassword != "" { + pemBytes, err = x509.DecryptPEMBlock(privPem, []byte(rsaPrivateKeyPassword)) + } else { + pemBytes = privPem.Bytes + } + + var parsedKey interface{} + if parsedKey, err = x509.ParsePKCS1PrivateKey(pemBytes); err != nil { + if parsedKey, err = x509.ParsePKCS8PrivateKey(pemBytes); err != nil { // note this returns type `interface{}` + utils.LogError("Unable to parse RSA private key, generating a temp one", err, utils.LogFields{}) + return GenRSA(4096) + } + } + + var privateKey *rsa.PrivateKey + var ok bool + if privateKey, ok = parsedKey.(*rsa.PrivateKey); !ok { + utils.LogError("Unable to parse RSA key, generating a temp one", err, utils.LogFields{}) + return GenRSA(4096) + } + + utils.LogInfo("Loaded RSA key from file", utils.LogFields{}) + return privateKey, nil +} + +// GenRSA returns a new RSA key of bits length +func GenRSA(bits int) (*rsa.PrivateKey, error) { + key, err := rsa.GenerateKey(rand.Reader, bits) + utils.LogFatal("Failed to generate signing key", err, nil) + return key, err +} diff --git a/modules/config/authconfig_test.go b/modules/config/authconfig_test.go new file mode 100644 index 0000000..0dfbc1d --- /dev/null +++ b/modules/config/authconfig_test.go @@ -0,0 +1,18 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestAuthConfigSetup(t *testing.T) { + SetConfigDir("../../config") + err := LoadConfig() + assert.NoError(t, err, "error initializing config needed for password") + + key, err := rsaConfigSetup("../../config/id_rsa", config.ServerConfig.RSAPrivateKeyPassword) + assert.Nil(t, err, "error loading rsa key") + assert.NotNil(t, key, "key was nil") + assert.NoError(t, key.Validate(), "key could not be validated") +} diff --git a/modules/config/manager.go b/modules/config/manager.go index d5b1982..73b3ea0 100644 --- a/modules/config/manager.go +++ b/modules/config/manager.go @@ -6,9 +6,9 @@ import ( "path/filepath" "time" - "github.com/CodeCollaborate/Server/utils" log "github.com/Sirupsen/logrus" - "github.com/kr/pretty" + + "github.com/CodeCollaborate/Server/utils" ) /** @@ -35,7 +35,8 @@ func LoadConfig() error { if err == nil { utils.LogInfo("Loaded Configuration", utils.LogFields{ - "ServerConfig": pretty.Sprint(config.ServerConfig), + //"ServerConfig": pretty.Sprint(config.ServerConfig), + // TODO: remove secret fields from config and then print again }) setLogLevel() } @@ -69,6 +70,10 @@ func setLogLevel() { } } +func init() { + log.SetFormatter(&log.TextFormatter{DisableColors: true}) +} + // EnableLoggingToFile redirects logger output to a logfile in the config's LogDir. // A new logfile will be created each time this method is called. func EnableLoggingToFile(logDir string) { @@ -77,6 +82,7 @@ func EnableLoggingToFile(logDir string) { logFile := filepath.Join(logDir, fmt.Sprintf("%d.%02d.%02d.%02d.%02d.log", time.Now().Year(), time.Now().Month(), time.Now().Day(), time.Now().Hour(), time.Now().Minute())) log.Infof("Logging to %s", logFile) + log.SetFormatter(&log.JSONFormatter{}) f, err := os.OpenFile(logFile, os.O_WRONLY|os.O_CREATE, 0755) if err != nil { log.Error("Failed to setup logging to file") diff --git a/modules/config/manager_test.go b/modules/config/manager_test.go index e13f672..0f40bcd 100644 --- a/modules/config/manager_test.go +++ b/modules/config/manager_test.go @@ -4,14 +4,17 @@ import ( "io/ioutil" "os" "path/filepath" - "reflect" "testing" + + "github.com/stretchr/testify/assert" ) func TestGetConfig(t *testing.T) { tmpDir := createTmpDir(t, ".", "test-config-files") defer os.RemoveAll(tmpDir) + // ensure configDir is set to the default. other tests could have run before and set it elsewhere + SetConfigDir("./config") err := LoadConfig() if err == nil { t.Fatal("Config dir not set yet; ./config does not exist. Should have failed.") @@ -42,10 +45,16 @@ func TestGetConfig(t *testing.T) { t.Fatal(err) } + assert.Nil(t, data.ServerConfig.rsaKey, "Ensure rsaKey lazily loaded") + privateKey := data.ServerConfig.RSAKey() + assert.NotNil(t, data.ServerConfig.rsaKey, "Ensure rsaKey lazily loaded") + assert.ObjectsAreEqual(privateKey, data.ServerConfig.rsaKey) + expected := &Config{ ServerConfig: ServerCfg{ - Name: "CodeCollaborate", - Port: 80, + Name: "CodeCollaborate", + Port: 80, + rsaKey: privateKey, // cheating }, ConnectionConfig: ConnCfgMap{ "MySQL": ConnCfg{ @@ -63,7 +72,5 @@ func TestGetConfig(t *testing.T) { }, } - if !reflect.DeepEqual(data, expected) { - t.Fatalf("Parsed data incorrect. Expected: \n%v\n Actual: \n%v\n", data, expected) - } + assert.ObjectsAreEqualValues(expected, data) } diff --git a/modules/config/models.go b/modules/config/models.go index 02ae7e8..65b7fa6 100644 --- a/modules/config/models.go +++ b/modules/config/models.go @@ -1,6 +1,11 @@ package config -import "time" +import ( + "crypto/rsa" + "time" + + "github.com/CodeCollaborate/Server/utils" +) /** * Models for the configuration CodeCollaborate Server. @@ -18,14 +23,21 @@ type Config struct { // ServerCfg contains various config items that pertain to the server type ServerCfg struct { Name string + Host string Port uint16 ProjectPath string DisableAuth bool + UseTLS bool LogLevel string TokenValidity string MinBufferLength int MaxBufferLength int + // RSA key + RSAPrivateKeyLocation string + RSAPrivateKeyPassword string + rsaKey *rsa.PrivateKey + // Parsed validity tokenValidityDuration time.Duration } @@ -41,6 +53,20 @@ func (cfg ServerCfg) TokenValidityDuration() (time.Duration, error) { return cfg.tokenValidityDuration, err } +// RSAKey returns the RSA key the server should use for signing tokens +func (cfg *ServerCfg) RSAKey() *rsa.PrivateKey { + if cfg.rsaKey != nil { + return cfg.rsaKey + } + + var err error + cfg.rsaKey, err = rsaConfigSetup(config.ServerConfig.RSAPrivateKeyLocation, config.ServerConfig.RSAPrivateKeyPassword) + if err != nil { + utils.LogFatal("Unable to load/generate RSA key", err, utils.LogFields{}) + } + return cfg.rsaKey +} + // ConnCfg represents the information required to make a connection type ConnCfg struct { Host string diff --git a/modules/datahandling/authentication.go b/modules/datahandling/authentication.go index f3c18ff..3f8fcef 100644 --- a/modules/datahandling/authentication.go +++ b/modules/datahandling/authentication.go @@ -1,13 +1,16 @@ package datahandling import ( + "crypto/rsa" "errors" "fmt" "strings" "time" - "github.com/CodeCollaborate/Server/modules/config" "github.com/dgrijalva/jwt-go" + + "github.com/CodeCollaborate/Server/modules/config" + "github.com/CodeCollaborate/Server/utils" ) type tokenPayload struct { @@ -16,6 +19,22 @@ type tokenPayload struct { Validity int64 } +var rsaKey *rsa.PrivateKey + +func getRsaKey() *rsa.PrivateKey { + if rsaKey != nil { + return rsaKey + } + + cfg := config.GetConfig() + if cfg == nil { + utils.LogFatal("Failed to load RSA key from config", errors.New("config not initialized"), utils.LogFields{}) + } + + rsaKey = cfg.ServerConfig.RSAKey() + return rsaKey +} + // Valid is the (unused) method to determine if the token is valid. however, since we need to have a reference // to the abstract request, we cannot do validation here. Token validation has been shifted to the authenticate // method. This is here for conformance to the token.Claims interface. @@ -26,10 +45,10 @@ func (tokenPayload) Valid() error { func authenticate(abs abstractRequest) error { token, err := jwt.ParseWithClaims(abs.SenderToken, &tokenPayload{}, func(token *jwt.Token) (interface{}, error) { // Don't forget to validate the alg is what you expect: - if _, ok := token.Method.(*jwt.SigningMethodECDSA); !ok { + if _, ok := token.Method.(*jwt.SigningMethodRSA); !ok { return nil, fmt.Errorf("ParseWithClaims - Unexpected signing method: %v", token.Header["alg"]) } - return &privKey.PublicKey, nil + return &getRsaKey().PublicKey, nil }) if err != nil { return fmt.Errorf("authenticate - failed to parse token: %s", err) @@ -40,7 +59,8 @@ func authenticate(abs abstractRequest) error { if !strings.EqualFold(claims.Username, abs.SenderID) { return errors.New("authenticate - senderID did not match token username") } - if time.Unix(claims.CreationTime, 0).After(time.Now()) { + // check if token is valid yet, but now with some wiggle room + if time.Unix(claims.CreationTime, 0).Add(-1 * time.Minute).After(time.Now()) { return errors.New("authenticate - token not valid yet") } if !time.Unix(claims.Validity, 0).After(time.Now()) { @@ -58,11 +78,11 @@ func newAuthToken(username string) (string, error) { return "", err } - token := jwt.NewWithClaims(jwt.SigningMethodES256, tokenPayload{ + token := jwt.NewWithClaims(jwt.SigningMethodRS512, tokenPayload{ Username: username, CreationTime: time.Now().Unix(), Validity: time.Now().Add(tokenValidityDuration).Unix(), }) - return token.SignedString(privKey) + return token.SignedString(getRsaKey()) } diff --git a/modules/datahandling/authentication_test.go b/modules/datahandling/authentication_test.go index 10b1d27..82a5a6a 100644 --- a/modules/datahandling/authentication_test.go +++ b/modules/datahandling/authentication_test.go @@ -1,7 +1,7 @@ package datahandling import ( - "crypto/ecdsa" + "crypto/rsa" "math/rand" "testing" "time" @@ -9,19 +9,25 @@ import ( "github.com/dgrijalva/jwt-go" "github.com/kr/pretty" "github.com/stretchr/testify/assert" + + "github.com/CodeCollaborate/Server/modules/config" ) func TestAuthenticateRandomUsernames(t *testing.T) { for i := 0; i < 100; i++ { username := randomString(20) - token := jwt.NewWithClaims(jwt.SigningMethodES256, tokenPayload{ + token := jwt.NewWithClaims(jwt.SigningMethodRS512, tokenPayload{ Username: username, CreationTime: time.Now().Unix(), Validity: time.Now().Add(1 * time.Hour).Unix(), }) - signed, err := token.SignedString(privKey) + key, err := config.GenRSA(1024) // make it small so it's faster + assert.Nil(t, err, "error generating rsa") + assert.NoError(t, key.Validate(), "Unable to validate RSA key") + + signed, err := token.SignedString(key) if err != nil { t.Fatal(err) } @@ -48,8 +54,8 @@ func TestAuthenticate(t *testing.T) { token: signedTokenOrDie(t, "TestUser1", time.Now().Unix(), - time.Now().Add(1*time.Second).Unix(), - privKey, + time.Now().Add(1*time.Minute).Unix(), + getRsaKey(), ), }, { @@ -58,8 +64,8 @@ func TestAuthenticate(t *testing.T) { token: signedTokenOrDie(t, "TestUser1", time.Now().Unix(), - time.Now().Add(1*time.Second).Unix(), - privKey, + time.Now().Add(1*time.Minute).Unix(), + getRsaKey(), ), }, { @@ -68,8 +74,8 @@ func TestAuthenticate(t *testing.T) { token: signedTokenOrDie(t, "TestUser1", time.Now().Unix(), - time.Now().Add(1*time.Second).Unix(), - privKey, + time.Now().Add(1*time.Minute).Unix(), + getRsaKey(), ), err: "authenticate - senderID did not match token username", }, @@ -79,8 +85,8 @@ func TestAuthenticate(t *testing.T) { token: signedTokenOrDie(t, "user1", time.Now().Unix(), - time.Now().Add(1*time.Second).Unix(), - privKey, + time.Now().Add(1*time.Minute).Unix(), + getRsaKey(), ), err: "authenticate - senderID did not match token username", }, @@ -90,8 +96,8 @@ func TestAuthenticate(t *testing.T) { token: signedTokenOrDie(t, "TestUser1", time.Now().Unix(), - time.Now().Add(-1*time.Second).Unix(), - privKey, + time.Now().Add(-1*time.Minute).Unix(), + getRsaKey(), ), err: "authenticate - expired token", }, @@ -100,9 +106,9 @@ func TestAuthenticate(t *testing.T) { senderID: "TestUser1", token: signedTokenOrDie(t, "TestUser1", - time.Now().Add(1*time.Second).Unix(), - time.Now().Add(1*time.Second).Unix(), - privKey, + time.Now().Add(1*time.Minute).Unix(), + time.Now().Add(1*time.Minute).Unix(), + getRsaKey(), ), err: "authenticate - token not valid yet", }, @@ -122,13 +128,13 @@ func TestAuthenticate(t *testing.T) { desc: "Wrong key", senderID: "TestUser1", token: "eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJVc2VybmFtZSI6IlRlc3RVc2VyMSIsIkNyZWF0aW9uVGltZSI6MTQ3NDQzOTQ2NywiVmFsaWRpdHkiOjE0NzQ0Mzk0NjZ9.6HK6VyBbXqIwJnRD2fCWIWTM6q466o56QhftJgcywawoi43kN-gEiwdx7K2EaGrDzxz9yd5jJHib_3n-_P9rxA", - err: "authenticate - failed to parse token: crypto/ecdsa: verification error", + err: "authenticate - failed to parse token: crypto/rsa: verification error", }, { desc: "Checksum mismatch", senderID: "TestUser1", token: "eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJVc2VybmFtZSI6IlRlc3RVc2VyMSIsIkNyZWF0aW9uVGltZSI6MTQ3NDQzOTQ2NywiVmFsaWRpdHkiOjE0NzQ0Mzk0NjZ9.6HK6VyBbXqIwJnRD2fCWIWTM6q466o56QhftJgcywawoi43kN-gEiwdx7K2EaGrDzxz9yd5jJw3b_3n-_P9rxA", - err: "authenticate - failed to parse token: crypto/ecdsa: verification error", + err: "authenticate - failed to parse token: crypto/rsa: verification error", }, } @@ -155,8 +161,8 @@ func TestAuthenticate(t *testing.T) { } } -func signedTokenOrDie(t *testing.T, username string, creationDate, validity int64, key *ecdsa.PrivateKey) string { - token := jwt.NewWithClaims(jwt.SigningMethodES256, tokenPayload{ +func signedTokenOrDie(t *testing.T, username string, creationDate, validity int64, key *rsa.PrivateKey) string { + token := jwt.NewWithClaims(jwt.SigningMethodRS512, tokenPayload{ Username: username, CreationTime: creationDate, Validity: validity, diff --git a/modules/datahandling/datahandler.go b/modules/datahandling/datahandler.go index 02eae64..994b6af 100644 --- a/modules/datahandling/datahandler.go +++ b/modules/datahandling/datahandler.go @@ -1,10 +1,7 @@ package datahandling import ( - "crypto/ecdsa" - "crypto/elliptic" - "crypto/rand" - "sync" + "strings" "github.com/CodeCollaborate/Server/modules/datahandling/messages" "github.com/CodeCollaborate/Server/modules/dbfs" @@ -12,15 +9,6 @@ import ( "github.com/CodeCollaborate/Server/utils" ) -var privKey *ecdsa.PrivateKey - -func init() { - key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - utils.LogFatal("Failed to generate signing key", err, nil) - - privKey = key -} - /** * Data Handling logic for the CodeCollaborate Server. */ @@ -28,23 +16,23 @@ func init() { // DataHandler handles the json data received from the WebSocket connection. type DataHandler struct { MessageChan chan<- rabbitmq.AMQPMessage - WebsocketID uint64 Db dbfs.DBFS } // Handle takes the MessageType and message in byte-array form, // processing the data, and updating DBFS/RabbitMQ as needed. -// the waitgroup allows the websocket manager to know when all requests have completed processing -func (dh DataHandler) Handle(messageType int, message []byte, wg *sync.WaitGroup) error { - defer wg.Done() - - utils.LogDebug("Received Message", utils.LogFields{ - "Message": string(message), - }) +func (dh DataHandler) Handle(message []byte, origin string, ack func() error) error { + // Ignore any request that has a password JSON field + if !strings.Contains(strings.ToLower(string(message)), "\"password\":") { + utils.LogDebug("Received Message", utils.LogFields{ + "Message": string(message), + }) + } req, err := createAbstractRequest(message) if err != nil { utils.LogError("Failed to parse json", err, nil) // Do not log request since passwords may be sent + ack() return err } @@ -52,34 +40,44 @@ func (dh DataHandler) Handle(messageType int, message []byte, wg *sync.WaitGroup fullRequest, err := getFullRequest(req) var closures []dhClosure - if err != nil { + // Ignore requests where there + if req.Resource == "User" && (req.Method == "Register" || req.Method == "Login") { + utils.LogError("getFullRequest failed for Register/Login", err, nil) + } else { + utils.LogError("getFullRequest failed", err, utils.LogFields{ + "Request": string(message), + }) + } if err == ErrAuthenticationFailed { utils.LogDebug("User not logged in", utils.LogFields{ "Resource": req.Resource, "Method": req.Method, + "SenderID": req.SenderID, + "Error": err, }) + ack() closures = []dhClosure{toSenderClosure{msg: messages.NewEmptyResponse(messages.StatusUnauthorized, req.Tag)}} } else { utils.LogDebug("No such resource/method", utils.LogFields{ "Resource": req.Resource, "Method": req.Method, }) + ack() closures = []dhClosure{toSenderClosure{msg: messages.NewEmptyResponse(messages.StatusUnimplemented, req.Tag)}} } } else { - closures, err = fullRequest.process(dh.Db) + closures, err = fullRequest.process(dh.Db, ack) if err != nil { utils.LogError("Failed to process request", err, utils.LogFields{ "Resource": req.Resource, "Method": req.Method, }) - // TODO: forward error message onto client? (or at least inform that error occurred) } } for _, closure := range closures { - err := closure.call(dh) + err := closure.call(dh, origin) if err != nil { utils.LogError("Failed to complete continuation", err, utils.LogFields{ "Resource": req.Resource, diff --git a/modules/datahandling/dhclosures.go b/modules/datahandling/dhclosures.go index 701fb24..f375a1a 100644 --- a/modules/datahandling/dhclosures.go +++ b/modules/datahandling/dhclosures.go @@ -10,7 +10,7 @@ import ( ) type dhClosure interface { - call(dh DataHandler) error + call(dh DataHandler, origin string) error } type toSenderClosure struct { @@ -18,7 +18,7 @@ type toSenderClosure struct { } // toSenderClosure.call is the function that will forward a server message back to the client -func (cont toSenderClosure) call(dh DataHandler) error { +func (cont toSenderClosure) call(dh DataHandler, origin string) error { msgJSON, err := json.Marshal(cont.msg) if err != nil { return err @@ -26,15 +26,19 @@ func (cont toSenderClosure) call(dh DataHandler) error { msg := rabbitmq.AMQPMessage{ Headers: map[string]interface{}{ - "Origin": rabbitmq.RabbitWebsocketQueueName(dh.WebsocketID), + "Origin": origin, "MessageType": cont.msg.Type, }, - RoutingKey: rabbitmq.RabbitWebsocketQueueName(dh.WebsocketID), + RoutingKey: origin, ContentType: rabbitmq.ContentTypeMsg, Persistent: false, Message: msgJSON, } + utils.LogDebug("Sending message to RabbitMQ client:", utils.LogFields{ + "Message": cont.msg, + "Key": msg.RoutingKey, + }) select { case dh.MessageChan <- msg: default: @@ -52,7 +56,7 @@ type toRabbitChannelClosure struct { } // toRabbitChannelClosure.call is the function that will forward a server message to a channel based on the given routing key -func (cont toRabbitChannelClosure) call(dh DataHandler) error { +func (cont toRabbitChannelClosure) call(dh DataHandler, origin string) error { msgJSON, err := json.Marshal(cont.msg) if err != nil { return err @@ -60,7 +64,7 @@ func (cont toRabbitChannelClosure) call(dh DataHandler) error { msg := rabbitmq.AMQPMessage{ Headers: map[string]interface{}{ - "Origin": rabbitmq.RabbitWebsocketQueueName(dh.WebsocketID), + "Origin": origin, "MessageType": cont.msg.Type, }, RoutingKey: cont.key, @@ -69,6 +73,10 @@ func (cont toRabbitChannelClosure) call(dh DataHandler) error { Message: msgJSON, } + utils.LogDebug("Sending message to RabbitMQ channel:", utils.LogFields{ + "Message": cont.msg, + "Key": msg.RoutingKey, + }) select { case dh.MessageChan <- msg: default: @@ -89,19 +97,19 @@ type rabbitCommandClosure struct { } // toRabbitChannelClosure.call is the function that will forward a server message to a channel based on the given routing key -func (cont rabbitCommandClosure) call(dh DataHandler) error { +func (cont rabbitCommandClosure) call(dh DataHandler, origin string) error { msgJSON, err := json.Marshal(cont) if err != nil { return err } if cont.Key == "" { - cont.Key = rabbitmq.RabbitWebsocketQueueName(dh.WebsocketID) + cont.Key = origin } msg := rabbitmq.AMQPMessage{ Headers: map[string]interface{}{ - "Origin": rabbitmq.RabbitWebsocketQueueName(dh.WebsocketID), + "Origin": origin, }, RoutingKey: cont.Key, ContentType: rabbitmq.ContentTypeCmd, diff --git a/modules/datahandling/dhmessages.go b/modules/datahandling/dhmessages.go index 0768112..48b6ed5 100644 --- a/modules/datahandling/dhmessages.go +++ b/modules/datahandling/dhmessages.go @@ -14,7 +14,7 @@ import ( // Request should be implemented by all request models. // Provides standard interface for calling the processing type request interface { - process(db dbfs.DBFS) (continuations []dhClosure, err error) + process(db dbfs.DBFS, ack func() error) (continuations []dhClosure, err error) setAbstractRequest(absReq *abstractRequest) } diff --git a/modules/datahandling/filerequests.go b/modules/datahandling/filerequests.go index 409691c..6028b4f 100644 --- a/modules/datahandling/filerequests.go +++ b/modules/datahandling/filerequests.go @@ -65,7 +65,9 @@ func (f *fileCreateRequest) setAbstractRequest(req *abstractRequest) { f.abstractRequest = *req } -func (f fileCreateRequest) process(db dbfs.DBFS) ([]dhClosure, error) { +func (f fileCreateRequest) process(db dbfs.DBFS, ack func() error) ([]dhClosure, error) { + defer ack() // ack regardless of success or failure + hasPermission, err := dbfs.PermissionAtLeast(f.SenderID, f.ProjectID, "write", db) if err != nil || !hasPermission { utils.LogError("API permission error", err, utils.LogFields{ @@ -131,7 +133,9 @@ func (f *fileRenameRequest) setAbstractRequest(req *abstractRequest) { f.abstractRequest = *req } -func (f fileRenameRequest) process(db dbfs.DBFS) ([]dhClosure, error) { +func (f fileRenameRequest) process(db dbfs.DBFS, ack func() error) ([]dhClosure, error) { + defer ack() // ack regardless of success or failure + fileMeta, err := db.MySQLFileGetInfo(f.FileID) if err != nil { return []dhClosure{toSenderClosure{msg: messages.NewEmptyResponse(messages.StatusFail, f.Tag)}}, err @@ -184,7 +188,9 @@ func (f *fileMoveRequest) setAbstractRequest(req *abstractRequest) { f.abstractRequest = *req } -func (f fileMoveRequest) process(db dbfs.DBFS) ([]dhClosure, error) { +func (f fileMoveRequest) process(db dbfs.DBFS, ack func() error) ([]dhClosure, error) { + defer ack() // ack regardless of success or failure + fileMeta, err := db.MySQLFileGetInfo(f.FileID) if err != nil { return []dhClosure{toSenderClosure{msg: messages.NewEmptyResponse(messages.StatusFail, f.Tag)}}, err @@ -236,7 +242,9 @@ func (f *fileDeleteRequest) setAbstractRequest(req *abstractRequest) { f.abstractRequest = *req } -func (f fileDeleteRequest) process(db dbfs.DBFS) ([]dhClosure, error) { +func (f fileDeleteRequest) process(db dbfs.DBFS, ack func() error) ([]dhClosure, error) { + defer ack() // ack regardless of success or failure + fileMeta, err := db.MySQLFileGetInfo(f.FileID) if err != nil { return []dhClosure{toSenderClosure{msg: messages.NewEmptyResponse(messages.StatusFail, f.Tag)}}, err @@ -285,7 +293,7 @@ func (f fileDeleteRequest) process(db dbfs.DBFS) ([]dhClosure, error) { // File.Change type fileChangeRequest struct { FileID int64 - Changes []string + Changes string abstractRequest } @@ -293,11 +301,12 @@ func (f *fileChangeRequest) setAbstractRequest(req *abstractRequest) { f.abstractRequest = *req } -func (f fileChangeRequest) process(db dbfs.DBFS) ([]dhClosure, error) { +func (f fileChangeRequest) process(db dbfs.DBFS, ack func() error) ([]dhClosure, error) { // This has to be before the CouchBase append, to make sure that the the two databases are kept in sync. // Specifically, this prevents CouchBase from incrementing a version number without the notifications being sent out. fileMeta, err := db.MySQLFileGetInfo(f.FileID) if err != nil { + ack() return []dhClosure{toSenderClosure{msg: messages.NewEmptyResponse(messages.StatusFail, f.Tag)}}, err } @@ -309,9 +318,13 @@ func (f fileChangeRequest) process(db dbfs.DBFS) ([]dhClosure, error) { "SenderID": f.SenderID, "ProjectID": fileMeta.ProjectID, }) + ack() return []dhClosure{toSenderClosure{msg: messages.NewEmptyResponse(messages.StatusUnauthorized, f.Tag)}}, nil } + // TODO: verify that acking before is the right decision + ack() // ack just before appending so if it fails in the middle it doesn't re-enqueue it + // TODO (normal/optional): verify changes are valid changes changes, version, missing, numchanges, err := db.CBAppendFileChange(fileMeta, f.Changes) if err != nil { @@ -328,7 +341,7 @@ func (f fileChangeRequest) process(db dbfs.DBFS) ([]dhClosure, error) { Tag: f.Tag, Data: struct { FileVersion int64 - Changes []string + Changes string MissingPatches []string }{ FileVersion: version, @@ -342,7 +355,7 @@ func (f fileChangeRequest) process(db dbfs.DBFS) ([]dhClosure, error) { ResourceID: f.FileID, Data: struct { FileVersion int64 - Changes []string + Changes string }{ FileVersion: version, Changes: changes, @@ -369,7 +382,9 @@ func (f *filePullRequest) setAbstractRequest(req *abstractRequest) { f.abstractRequest = *req } -func (f filePullRequest) process(db dbfs.DBFS) ([]dhClosure, error) { +func (f filePullRequest) process(db dbfs.DBFS, ack func() error) ([]dhClosure, error) { + defer ack() // ack regardless of success or failure + fileMeta, err := db.MySQLFileGetInfo(f.FileID) if err != nil { return []dhClosure{toSenderClosure{msg: messages.NewEmptyResponse(messages.StatusFail, f.Tag)}}, err diff --git a/modules/datahandling/filerequests_test.go b/modules/datahandling/filerequests_test.go index cd61bb8..f95bfd1 100644 --- a/modules/datahandling/filerequests_test.go +++ b/modules/datahandling/filerequests_test.go @@ -36,11 +36,16 @@ func TestFileCreateRequest_Process(t *testing.T) { db.FunctionCallCount = 0 - closures, err := req.process(db) - if err != nil { - t.Fatal(err) + acked := false + ack := func() error { + acked = !acked + return nil } + closures, err := req.process(db, ack) + assert.Nil(t, err) + assert.True(t, acked, "process function did not ack message") + // didn't call extra db functions assert.Equal(t, 4, db.FunctionCallCount, "did not call correct number of db functions") @@ -87,11 +92,16 @@ func TestFileRenameRequest_Process(t *testing.T) { db.FunctionCallCount = 0 - closures, err := req.process(db) - if err != nil { - t.Fatal(err) + acked := false + ack := func() error { + acked = !acked + return nil } + closures, err := req.process(db, ack) + assert.Nil(t, err) + assert.True(t, acked, "process function did not ack message") + // didn't call extra db functions assert.Equal(t, 4, db.FunctionCallCount, "did not call correct number of db functions") @@ -144,11 +154,16 @@ func TestFileMoveRequest_Process(t *testing.T) { db.FunctionCallCount = 0 - closures, err := req.process(db) - if err != nil { - t.Fatal(err) + acked := false + ack := func() error { + acked = !acked + return nil } + closures, err := req.process(db, ack) + assert.Nil(t, err) + assert.True(t, acked, "process function did not ack message") + // didn't call extra db functions assert.Equal(t, 4, db.FunctionCallCount, "did not call correct number of db functions") @@ -200,11 +215,16 @@ func TestFileDeleteRequest_Process(t *testing.T) { db.FunctionCallCount = 0 - closures, err := req.process(db) - if err != nil { - t.Fatal(err) + acked := false + ack := func() error { + acked = !acked + return nil } + closures, err := req.process(db, ack) + assert.Nil(t, err) + assert.True(t, acked, "process function did not ack message") + // didn't call extra db functions assert.Equal(t, 5, db.FunctionCallCount, "did not call correct number of db functions") @@ -251,17 +271,21 @@ func TestFileChangeRequest_Process(t *testing.T) { req.Resource = "File" req.Method = "Change" req.FileID = fileid - req.Changes = []string{"v0:\n0:+1:a"} - + req.Changes = "v0:\n0:+1:a:\n10" baseFileVersion := int64(1) db.FunctionCallCount = 0 - closures, err := req.process(db) - if err != nil { - t.Fatal(err) + acked := false + ack := func() error { + acked = !acked + return nil } + closures, err := req.process(db, ack) + assert.Nil(t, err) + assert.True(t, acked, "process function did not ack message") + // didn't call extra db functions assert.Equal(t, 3, db.FunctionCallCount, "did not call correct number of db functions") @@ -288,12 +312,12 @@ func TestFileChangeRequest_Process(t *testing.T) { t.Fatal("wrong FileID recieved in notification") } - changes := reflect.ValueOf(closure.msg.ServerMessage.(messages.Notification).Data).FieldByName("Changes").Interface().([]string) - if changes[0] != req.Changes[0] { + changes := reflect.ValueOf(closure.msg.ServerMessage.(messages.Notification).Data).FieldByName("Changes").Interface().(string) + if changes != req.Changes { t.Fatal("wrong changes recieved in notification") } - if db.FileChanges[fileid][0] != changes[0] { + if db.FileChanges[fileid][0] != changes { t.Fatal("changes not inserted") } @@ -304,14 +328,19 @@ func TestFileChangeRequest_Process(t *testing.T) { // try the request again to prove that it rejects higher file versions - req.Changes = []string{"v9999:\n0:+1:a"} + req.Changes = "v9999:\n0:+1:a:\n10" db.FunctionCallCount = 0 - closures, err = req.process(db) - if err != dbfs.ErrVersionOutOfDate { - t.Fatal(err) + acked = false + ack = func() error { + acked = !acked + return nil } + closures, err = req.process(db, ack) + assert.Equal(t, dbfs.ErrVersionOutOfDate, err, "expected version to be out of date") + assert.True(t, acked, "process function did not ack message") + // didn't call extra db functions assert.Equal(t, 3, db.FunctionCallCount, "did not call correct number of db functions") @@ -340,7 +369,7 @@ func TestFilePullRequest_Process(t *testing.T) { fileid, err := db.MySQLFileCreate("loganga", "new file", "", projectID) db.FileWrite("./", "new file", projectID, []byte{}) - changes := []string{"v0:\n0:+1:a"} + changes := "v0:\n0:+1:a:\n10" db.CBAppendFileChange(dbfs.FileMeta{FileID: fileid}, changes) req.Resource = "File" @@ -349,11 +378,16 @@ func TestFilePullRequest_Process(t *testing.T) { db.FunctionCallCount = 0 - closures, err := req.process(db) - if err != nil { - t.Fatal(err) + acked := false + ack := func() error { + acked = !acked + return nil } + closures, err := req.process(db, ack) + assert.Nil(t, err) + assert.True(t, acked, "process function did not ack message") + // didn't call extra db functions if db.FunctionCallCount != 3 { t.Fatal("did not call correct number of db functions") @@ -373,7 +407,7 @@ func TestFilePullRequest_Process(t *testing.T) { // is the data actually correct fileChanges := reflect.ValueOf(resp.Data).FieldByName("Changes").Interface().([]string) - if changes[0] != fileChanges[0] { + if changes != fileChanges[0] { t.Fatalf("wrong file changes, expected: %v, got: %v", changes, fileChanges) } } diff --git a/modules/datahandling/projectrequests.go b/modules/datahandling/projectrequests.go index 2ed0a50..775fada 100644 --- a/modules/datahandling/projectrequests.go +++ b/modules/datahandling/projectrequests.go @@ -77,7 +77,9 @@ func (p *projectCreateRequest) setAbstractRequest(req *abstractRequest) { p.abstractRequest = *req } -func (p projectCreateRequest) process(db dbfs.DBFS) ([]dhClosure, error) { +func (p projectCreateRequest) process(db dbfs.DBFS, ack func() error) ([]dhClosure, error) { + ack() // ack regardless of success or failure + projectID, err := db.MySQLProjectCreate(p.SenderID, p.Name) if err != nil { //if err == project already exists { @@ -110,7 +112,9 @@ func (p *projectRenameRequest) setAbstractRequest(req *abstractRequest) { p.abstractRequest = *req } -func (p projectRenameRequest) process(db dbfs.DBFS) ([]dhClosure, error) { +func (p projectRenameRequest) process(db dbfs.DBFS, ack func() error) ([]dhClosure, error) { + defer ack() // ack regardless of success or failure + hasPermission, err := dbfs.PermissionAtLeast(p.SenderID, p.ProjectID, "write", db) if err != nil || !hasPermission { utils.LogError("API permission error", err, utils.LogFields{ @@ -151,7 +155,9 @@ func (p *projectGetPermissionConstantsRequest) setAbstractRequest(req *abstractR p.abstractRequest = *req } -func (p projectGetPermissionConstantsRequest) process(db dbfs.DBFS) ([]dhClosure, error) { +func (p projectGetPermissionConstantsRequest) process(db dbfs.DBFS, ack func() error) ([]dhClosure, error) { + defer ack() // ack regardless of success or failure + res := messages.Response{ Status: messages.StatusSuccess, Tag: p.Tag, @@ -173,7 +179,9 @@ type projectGrantPermissionsRequest struct { abstractRequest } -func (p projectGrantPermissionsRequest) process(db dbfs.DBFS) ([]dhClosure, error) { +func (p projectGrantPermissionsRequest) process(db dbfs.DBFS, ack func() error) ([]dhClosure, error) { + defer ack() // ack regardless of success or failure + hasPermission, err := dbfs.PermissionAtLeast(p.SenderID, p.ProjectID, "admin", db) if err != nil || !hasPermission { utils.LogError("API permission error", err, utils.LogFields{ @@ -185,6 +193,13 @@ func (p projectGrantPermissionsRequest) process(db dbfs.DBFS) ([]dhClosure, erro return []dhClosure{toSenderClosure{msg: messages.NewEmptyResponse(messages.StatusUnauthorized, p.Tag)}}, nil } + // Prevent users from changing their own permissions + if p.SenderID == p.GrantUsername { + return []dhClosure{toSenderClosure{msg: messages.NewEmptyResponse(messages.StatusUnauthorized, p.Tag)}}, nil + } + + // TODO: Add if User exists check + requestPerm, err := config.PermissionByLevel(p.PermissionLevel) if err != nil { return []dhClosure{toSenderClosure{msg: messages.NewEmptyResponse(messages.StatusFail, p.Tag)}}, nil @@ -236,7 +251,9 @@ type projectRevokePermissionsRequest struct { abstractRequest } -func (p projectRevokePermissionsRequest) process(db dbfs.DBFS) ([]dhClosure, error) { +func (p projectRevokePermissionsRequest) process(db dbfs.DBFS, ack func() error) ([]dhClosure, error) { + defer ack() // ack regardless of success or failure + hasPermission, err := dbfs.PermissionAtLeast(p.SenderID, p.ProjectID, "admin", db) if err != nil { utils.LogError("API permission error", err, utils.LogFields{ @@ -318,7 +335,9 @@ type projectGetOnlineClientsRequest struct { abstractRequest } -func (p projectGetOnlineClientsRequest) process(db dbfs.DBFS) ([]dhClosure, error) { +func (p projectGetOnlineClientsRequest) process(db dbfs.DBFS, ack func() error) ([]dhClosure, error) { + defer ack() // ack regardless of success or failure + // TODO: implement on redis (and actually implement redis) utils.LogWarn("ProjectGetOnlineClients not implemented", nil) @@ -342,7 +361,9 @@ type projectLookupResult struct { Permissions map[string](dbfs.ProjectPermission) } -func (p projectLookupRequest) process(db dbfs.DBFS) ([]dhClosure, error) { +func (p projectLookupRequest) process(db dbfs.DBFS, ack func() error) ([]dhClosure, error) { + defer ack() // ack regardless of success or failure + /* We could do data := make([]interface{}, len(p.ProjectIDs)) @@ -452,7 +473,9 @@ type fileLookupResult struct { Version int64 } -func (p projectGetFilesRequest) process(db dbfs.DBFS) ([]dhClosure, error) { +func (p projectGetFilesRequest) process(db dbfs.DBFS, ack func() error) ([]dhClosure, error) { + defer ack() // ack regardless of success or failure + hasPermission, err := dbfs.PermissionAtLeast(p.SenderID, p.ProjectID, "read", db) if err != nil || !hasPermission { utils.LogError("API permission error", err, utils.LogFields{ @@ -548,7 +571,9 @@ type projectSubscribeRequest struct { abstractRequest } -func (p projectSubscribeRequest) process(db dbfs.DBFS) ([]dhClosure, error) { +func (p projectSubscribeRequest) process(db dbfs.DBFS, ack func() error) ([]dhClosure, error) { + defer ack() // ack regardless of success or failure + hasPermission, err := dbfs.PermissionAtLeast(p.SenderID, p.ProjectID, "read", db) if err != nil || !hasPermission { utils.LogError("API permission error", err, utils.LogFields{ @@ -580,7 +605,9 @@ type projectUnsubscribeRequest struct { abstractRequest } -func (p projectUnsubscribeRequest) process(db dbfs.DBFS) ([]dhClosure, error) { +func (p projectUnsubscribeRequest) process(db dbfs.DBFS, ack func() error) ([]dhClosure, error) { + defer ack() // ack regardless of success or failure + cmdClosure := rabbitCommandClosure{ Command: "Unsubscribe", Tag: p.Tag, @@ -601,7 +628,9 @@ type projectDeleteRequest struct { abstractRequest } -func (p projectDeleteRequest) process(db dbfs.DBFS) ([]dhClosure, error) { +func (p projectDeleteRequest) process(db dbfs.DBFS, ack func() error) ([]dhClosure, error) { + defer ack() // ack regardless of success or failure + hasPermission, err := dbfs.PermissionAtLeast(p.SenderID, p.ProjectID, "owner", db) if err != nil { utils.LogError("API permission error", err, utils.LogFields{ @@ -632,7 +661,9 @@ func (p projectDeleteRequest) process(db dbfs.DBFS) ([]dhClosure, error) { RevokeUsername: p.SenderID, abstractRequest: p.abstractRequest, } - return realRequest.process(db) + return realRequest.process(db, func() error { + return nil // orig request already has a deferred ack + }) } return []dhClosure{toSenderClosure{msg: messages.NewEmptyResponse(messages.StatusUnauthorized, p.Tag)}}, nil diff --git a/modules/datahandling/projectrequests_test.go b/modules/datahandling/projectrequests_test.go index 2a8f6d1..20e7649 100644 --- a/modules/datahandling/projectrequests_test.go +++ b/modules/datahandling/projectrequests_test.go @@ -4,11 +4,12 @@ import ( "reflect" "testing" + "github.com/stretchr/testify/assert" + "github.com/CodeCollaborate/Server/modules/config" "github.com/CodeCollaborate/Server/modules/datahandling/messages" "github.com/CodeCollaborate/Server/modules/dbfs" "github.com/CodeCollaborate/Server/modules/rabbitmq" - "github.com/stretchr/testify/assert" ) func setBaseFields(req request) { @@ -18,11 +19,6 @@ func setBaseFields(req request) { }) } -var datahanly = DataHandler{ - MessageChan: make(chan rabbitmq.AMQPMessage, 1), - WebsocketID: 1, -} - func TestProjectCreateRequest_Process(t *testing.T) { configSetup(t) req := *new(projectCreateRequest) @@ -35,11 +31,16 @@ func TestProjectCreateRequest_Process(t *testing.T) { db := dbfs.NewDBMock() db.Users["loganga"] = geneMeta - closures, err := req.process(db) - if err != nil { - t.Fatal(err) + acked := false + ack := func() error { + acked = !acked + return nil } + closures, err := req.process(db, ack) + assert.Nil(t, err) + assert.True(t, acked, "process function did not ack message") + // didn't call extra db functions if db.FunctionCallCount != 1 { t.Fatal("did not call correct number of db functions") @@ -93,11 +94,16 @@ func TestProjectRenameRequest_Process(t *testing.T) { db.Projects["loganga"] = []dbfs.ProjectMeta{projectmeta} db.ProjectIDCounter = 2 - closures, err := req.process(db) - if err != nil { - t.Fatal(err) + acked := false + ack := func() error { + acked = !acked + return nil } + closures, err := req.process(db, ack) + assert.Nil(t, err) + assert.True(t, acked, "process function did not ack message") + // didn't call extra db functions assert.Equal(t, 2, db.FunctionCallCount, "did not call correct number of db functions") @@ -128,8 +134,16 @@ func TestProjectGetPermissionConstantsRequest_Process(t *testing.T) { setBaseFields(&req) db := dbfs.NewDBMock() - closures, err := req.process(db) + acked := false + ack := func() error { + acked = !acked + return nil + } + + closures, err := req.process(db, ack) assert.Nil(t, err) + assert.True(t, acked, "process function did not ack message") + assert.Zero(t, db.FunctionCallCount, "unexpected db calls for permission constants") assert.Equal(t, 1, len(closures), "unexpected number of returned closures") @@ -176,11 +190,16 @@ func TestProjectGrantPermissionsRequest_Process(t *testing.T) { db.FunctionCallCount = 0 req.ProjectID = projectID - closures, err := req.process(db) - if err != nil { - t.Fatal(err) + acked := false + ack := func() error { + acked = !acked + return nil } + closures, err := req.process(db, ack) + assert.Nil(t, err) + assert.True(t, acked, "process function did not ack message") + // didn't call extra db functions assert.Equal(t, 2, db.FunctionCallCount, "did not call correct number of db functions") @@ -238,11 +257,16 @@ func TestProjectRevokePermissionsRequest_Process(t *testing.T) { db.FunctionCallCount = 0 req.ProjectID = projectID - closures, err := req.process(db) - if err != nil { - t.Fatal(err) + acked := false + ack := func() error { + acked = !acked + return nil } + closures, err := req.process(db, ack) + assert.Nil(t, err) + assert.True(t, acked, "process function did not ack message") + // didn't call extra db functions assert.Equal(t, 2, db.FunctionCallCount, "did not call correct number of db functions") @@ -323,11 +347,16 @@ func TestProjectLookupRequest_Process(t *testing.T) { req.ProjectIDs = []int64{projid1, projid2} db.FunctionCallCount = 0 - closures, err := req.process(db) - if err != nil { - t.Fatal(err) + acked := false + ack := func() error { + acked = !acked + return nil } + closures, err := req.process(db, ack) + assert.Nil(t, err) + assert.True(t, acked, "process function did not ack message") + // didn't call extra db functions assert.Equal(t, 4, db.FunctionCallCount, "did not call correct number of db functions") @@ -374,11 +403,16 @@ func TestProjectGetFilesRequest_Process(t *testing.T) { req.ProjectID = projid1 db.FunctionCallCount = 0 - closures, err := req.process(db) - if err != nil { - t.Fatal(err) + acked := false + ack := func() error { + acked = !acked + return nil } + closures, err := req.process(db, ack) + assert.Nil(t, err) + assert.True(t, acked, "process function did not ack message") + // didn't call extra db functions assert.Equal(t, 5, db.FunctionCallCount, "did not call correct number of db functions") @@ -421,11 +455,16 @@ func TestProjectSubscribe_Process(t *testing.T) { db.FunctionCallCount = 0 - closures, err := req.process(db) - if err != nil { - t.Fatal(err) + acked := false + ack := func() error { + acked = !acked + return nil } + closures, err := req.process(db, ack) + assert.Nil(t, err) + assert.True(t, acked, "process function did not ack message") + // are we notifying the right people if len(closures) != 1 || reflect.TypeOf(closures[0]).String() != "datahandling.rabbitCommandClosure" { @@ -450,11 +489,16 @@ func TestProjectUnsubscribe_Process(t *testing.T) { req.Method = "Unsubscribe" req.ProjectID = 1 - closures, err := req.process(db) - if err != nil { - t.Fatal(err) + acked := false + ack := func() error { + acked = !acked + return nil } + closures, err := req.process(db, ack) + assert.Nil(t, err) + assert.True(t, acked, "process function did not ack message") + // are we notifying the right people if len(closures) != 1 || reflect.TypeOf(closures[0]).String() != "datahandling.rabbitCommandClosure" { @@ -484,11 +528,16 @@ func TestProjectDeleteRequest_process(t *testing.T) { db.FunctionCallCount = 0 req.ProjectID = projID - closures, err := req.process(db) - if err != nil { - t.Fatal(err) + acked := false + ack := func() error { + acked = !acked + return nil } + closures, err := req.process(db, ack) + assert.Nil(t, err) + assert.True(t, acked, "process function did not ack message") + // didn't call extra db functions assert.Equal(t, 2, db.FunctionCallCount, "did not call correct number of db functions") @@ -538,11 +587,16 @@ func TestProjectDeleteTurnsIntoRevokeRequest(t *testing.T) { req.ProjectID = projectID req.SenderID = notgenemeta.Username - closures, err := req.process(db) - if err != nil { - t.Fatal(err) + acked := false + ack := func() error { + acked = !acked + return nil } + closures, err := req.process(db, ack) + assert.Nil(t, err) + assert.True(t, acked, "process function did not ack message") + // didn't call extra db functions assert.Equal(t, 4, db.FunctionCallCount, "did not call correct number of db functions") diff --git a/modules/datahandling/requestmap.go b/modules/datahandling/requestmap.go index fd34e92..6c325cb 100644 --- a/modules/datahandling/requestmap.go +++ b/modules/datahandling/requestmap.go @@ -29,10 +29,11 @@ func getFullRequest(req *abstractRequest) (request, error) { } // authenticated request - if config.GetConfig().ServerConfig.DisableAuth || authenticate(*req) == nil { + err := authenticate(*req) + if config.GetConfig().ServerConfig.DisableAuth || err == nil { return authenticatedRequest(req) } - return nil, ErrAuthenticationFailed + return nil, err } // authenticatedRequest returns fully parsed Request from the given authenticated AbstractRequest diff --git a/modules/datahandling/requestmap_test.go b/modules/datahandling/requestmap_test.go index dd54af0..8db1e2c 100644 --- a/modules/datahandling/requestmap_test.go +++ b/modules/datahandling/requestmap_test.go @@ -332,7 +332,7 @@ func TestFileChangeRequest(t *testing.T) { req.Data = json.RawMessage("{" + "\"FileID\": 12345," + "\"FileVersion\": 25," + - "\"Changes\": [\"ok\", \"k\"]" + + "\"Changes\": \"ok\"" + "}") newRequest, err := getFullRequest(&req) diff --git a/modules/datahandling/testutils_test.go b/modules/datahandling/testutils_test.go index 4c4a25d..584a13b 100644 --- a/modules/datahandling/testutils_test.go +++ b/modules/datahandling/testutils_test.go @@ -16,5 +16,8 @@ func configSetup(t *testing.T) { } func testToken(t *testing.T, username string) string { - return signedTokenOrDie(t, username, time.Now().Unix(), time.Now().Add(1*time.Minute).Unix(), privKey) + rsa, _ := config.GenRSA(1024) // make it small so it's faster for tests + // fun story: I originally had this set to 4096 by default and accidentally + // made the tests take 10 minutes + return signedTokenOrDie(t, username, time.Now().Unix(), time.Now().Add(1*time.Minute).Unix(), rsa) } diff --git a/modules/datahandling/userrequests.go b/modules/datahandling/userrequests.go index cd1968a..2acc99f 100644 --- a/modules/datahandling/userrequests.go +++ b/modules/datahandling/userrequests.go @@ -1,11 +1,12 @@ package datahandling import ( + "golang.org/x/crypto/bcrypt" + "github.com/CodeCollaborate/Server/modules/datahandling/messages" "github.com/CodeCollaborate/Server/modules/dbfs" "github.com/CodeCollaborate/Server/modules/rabbitmq" "github.com/CodeCollaborate/Server/utils" - "golang.org/x/crypto/bcrypt" ) var userRequestsSetup = false @@ -53,7 +54,8 @@ func (f *userRegisterRequest) setAbstractRequest(req *abstractRequest) { f.abstractRequest = *req } -func (f userRegisterRequest) process(db dbfs.DBFS) ([]dhClosure, error) { +func (f userRegisterRequest) process(db dbfs.DBFS, ack func() error) ([]dhClosure, error) { + defer ack() // ack regardless of success or failure hashed, err := bcrypt.GenerateFromPassword([]byte(f.Password), bcrypt.DefaultCost) if err != nil { @@ -92,7 +94,9 @@ func (f *userLoginRequest) setAbstractRequest(req *abstractRequest) { f.abstractRequest = *req } -func (f userLoginRequest) process(db dbfs.DBFS) ([]dhClosure, error) { +func (f userLoginRequest) process(db dbfs.DBFS, ack func() error) ([]dhClosure, error) { + defer ack() // ack regardless of success or failure + hashed, err := db.MySQLUserGetPass(f.Username) if err != nil { return []dhClosure{toSenderClosure{msg: messages.NewEmptyResponse(messages.StatusFail, f.Tag)}}, err @@ -143,7 +147,9 @@ func (f *userDeleteRequest) setAbstractRequest(req *abstractRequest) { f.abstractRequest = *req } -func (f userDeleteRequest) process(db dbfs.DBFS) ([]dhClosure, error) { +func (f userDeleteRequest) process(db dbfs.DBFS, ack func() error) ([]dhClosure, error) { + defer ack() // ack regardless of success or failure + deletedIDs, err := db.MySQLUserDelete(f.SenderID) if err != nil { @@ -176,7 +182,9 @@ func (f *userLookupRequest) setAbstractRequest(req *abstractRequest) { f.abstractRequest = *req } -func (f userLookupRequest) process(db dbfs.DBFS) ([]dhClosure, error) { +func (f userLookupRequest) process(db dbfs.DBFS, ack func() error) ([]dhClosure, error) { + defer ack() // ack regardless of success or failure + users := make([]dbfs.UserMeta, len(f.Usernames)) index := 0 var erro error @@ -232,7 +240,9 @@ func (f *userProjectsRequest) setAbstractRequest(req *abstractRequest) { f.abstractRequest = *req } -func (f userProjectsRequest) process(db dbfs.DBFS) ([]dhClosure, error) { +func (f userProjectsRequest) process(db dbfs.DBFS, ack func() error) ([]dhClosure, error) { + defer ack() // ack regardless of success or failure + var errOut error projects, errOut := db.MySQLUserProjects(f.SenderID) diff --git a/modules/datahandling/userrequests_test.go b/modules/datahandling/userrequests_test.go index 2c1994e..433d1e1 100644 --- a/modules/datahandling/userrequests_test.go +++ b/modules/datahandling/userrequests_test.go @@ -4,10 +4,11 @@ import ( "reflect" "testing" + "github.com/stretchr/testify/assert" + "github.com/CodeCollaborate/Server/modules/config" "github.com/CodeCollaborate/Server/modules/datahandling/messages" "github.com/CodeCollaborate/Server/modules/dbfs" - "github.com/stretchr/testify/assert" ) func TestUserRegisterRequest_Process(t *testing.T) { @@ -25,13 +26,17 @@ func TestUserRegisterRequest_Process(t *testing.T) { req.Password = "correct horse battery staple" db := dbfs.NewDBMock() - datahanly.Db = db - closures, err := req.process(db) - if err != nil { - t.Fatal(err) + acked := false + ack := func() error { + acked = !acked + return nil } + closures, err := req.process(db, ack) + assert.Nil(t, err) + assert.True(t, acked, "process function did not ack message") + // didn't call extra db functions if db.FunctionCallCount != 1 { t.Fatal("did not call correct number of db functions") @@ -52,10 +57,15 @@ func TestUserRegisterRequest_Process(t *testing.T) { t.Fatalf("Process function responded with status: %d", cont) } - closures, err = req.process(db) - if err == nil { - t.Fatal("Should have failed to register user that already exists") + acked = false + ack = func() error { + acked = !acked + return nil } + + closures, err = req.process(db, ack) + assert.NotNil(t, err, "Should have failed to register user that already exists") + assert.True(t, acked, "process function did not ack message") } // userLoginRequest.process is unimplemented @@ -73,8 +83,16 @@ func TestUserDeleteRequest_Process(t *testing.T) { db.MySQLUserRegister(geneMeta) db.FunctionCallCount = 0 - closures, err := req.process(db) + acked := false + ack := func() error { + acked = !acked + return nil + } + + closures, err := req.process(db, ack) assert.Nil(t, err) + assert.True(t, acked, "process function did not ack message") + assert.Equal(t, 2, db.FunctionCallCount, "unexpected db calls for user delete") assert.Equal(t, 1, len(closures), "unexpected number of returned closures") @@ -98,8 +116,16 @@ func TestUserDeleteRequest_Process(t *testing.T) { db.FunctionCallCount = 0 - closures, err = req.process(db) + acked = false + ack = func() error { + acked = !acked + return nil + } + + closures, err = req.process(db, ack) assert.Nil(t, err) + assert.True(t, acked, "process function did not ack message") + assert.Equal(t, 2, db.FunctionCallCount, "unexpected db calls for user delete") assert.Equal(t, 3, len(closures), "unexpected number of returned closures") @@ -141,11 +167,16 @@ func TestUserLookupRequest_Process(t *testing.T) { } db.Users["loganga"] = meta - closures, err := req.process(db) - if err != nil { - t.Fatal(err) + acked := false + ack := func() error { + acked = !acked + return nil } + closures, err := req.process(db, ack) + assert.Nil(t, err) + assert.True(t, acked, "process function did not ack message") + // didn't call extra db functions if db.FunctionCallCount != 1 { t.Fatal("did not call correct number of db functions") @@ -203,11 +234,16 @@ func TestUserProjectsRequest_Process(t *testing.T) { db.FunctionCallCount = 0 - closures, err := req.process(db) - if err != nil { - t.Fatal(err) + acked := false + ack := func() error { + acked = !acked + return nil } + closures, err := req.process(db, ack) + assert.Nil(t, err) + assert.True(t, acked, "process function did not ack message") + // didn't call extra db functions if db.FunctionCallCount != 2 { t.Fatalf("did not call correct number of db functions, called %d # of arguments", db.FunctionCallCount) @@ -239,11 +275,16 @@ func TestUserProjectsRequest_Process(t *testing.T) { db.FunctionCallCount = 0 - closures, err = req.process(db) - if err != nil { - t.Fatal(err) + acked = false + ack = func() error { + acked = !acked + return nil } + closures, err = req.process(db, ack) + assert.Nil(t, err) + assert.True(t, acked, "process function did not ack message") + // are we notifying the right people if len(closures) != 1 || reflect.TypeOf(closures[0]).String() != "datahandling.toSenderClosure" { diff --git a/modules/dbfs/couchbase.go b/modules/dbfs/couchbase.go index 43cc8d8..c005a56 100644 --- a/modules/dbfs/couchbase.go +++ b/modules/dbfs/couchbase.go @@ -10,6 +10,7 @@ import ( "github.com/CodeCollaborate/Server/modules/patching" "github.com/CodeCollaborate/Server/utils" "github.com/couchbase/gocb" + "github.com/davecgh/go-spew/spew" ) type couchbaseConn struct { @@ -155,18 +156,26 @@ func (di *DatabaseImpl) CBGetFileVersion(fileID int64) (int64, error) { // CBAppendFileChange mutates the file document with the new change and sets the new version number // Returns the new version number, the missing patches, the total count of patches tracked, and an error, if any. -func (di *DatabaseImpl) CBAppendFileChange(fileMeta FileMeta, patches []string) ([]string, int64, []string, int, error) { +func (di *DatabaseImpl) CBAppendFileChange(fileMeta FileMeta, patchStr string) (string, int64, []string, int, error) { cb, err := di.openCouchBase() if err != nil { - return nil, -1, nil, 0, err + return "", -1, nil, 0, err } // optimistic locking operation // check the version is accurate and get the object's cas, // then use it in the MutateIn call to verify the document hasn't updated underneath us - prevChanges, cas, version, useTemp, err := di.PullChanges(fileMeta) + prevChangeStrs, cas, version, useTemp, err := di.PullChanges(fileMeta) if err != nil { - return nil, -1, nil, 0, err + return "", -1, nil, 0, err + } + + prevChanges, err := patching.GetPatches(prevChangeStrs) + if err != nil { + utils.LogError("Failed to parse previous changes into patch objects", err, utils.LogFields{ + "PrevChanges": prevChangeStrs, + }) + return "", -1, nil, 0, err } if cas == uint64(0) { @@ -177,175 +186,149 @@ func (di *DatabaseImpl) CBAppendFileChange(fileMeta FileMeta, patches []string) } minVersion := version - if len(prevChanges) > 0 { - startPatch, err := patching.NewPatchFromString(prevChanges[0]) + if len(prevChangeStrs) > 0 { + startPatch, err := patching.NewPatchFromString(prevChangeStrs[0]) if err != nil { - return nil, -1, nil, 0, ErrInternalServerError + utils.LogError("Failed to parse first patch", err, utils.LogFields{ + "PatchStr": prevChangeStrs[0], + }) + return "", -1, nil, 0, ErrInternalServerError } // Allow transform-patches to start on the same base version as the head (after linearization, we have all the necessary patches) minVersion = startPatch.BaseVersion } minStartIndex := int64(math.MaxInt64) - transformedPatches := []string{} - prevChangesCopy := make([]string, len(prevChanges)) - copy(prevChangesCopy, prevChanges) + prevChangesCopy := make([]string, len(prevChangeStrs)) + copy(prevChangesCopy, prevChangeStrs) // Build patch, transform changes against newer changes. - for _, changeStr := range patches { - change, err := patching.NewPatchFromString(changeStr) - if err != nil { - return nil, -1, nil, 0, errors.New("Failed to parse patch") - } + change, err := patching.NewPatchFromString(patchStr) + if err != nil { + return "", -1, nil, 0, errors.New("Failed to parse patch") + } - // For every patch, calculate the patches that it does not have. - utils.LogDebug("CHANGES VERSIONS", utils.LogFields{ - "Version": version, - "BaseVersion": change.BaseVersion, - "Diff": int(version - change.BaseVersion), - "Len": len(prevChanges), - "ChangeStr": changeStr, - "PrevChanges": prevChanges, - "minVersion": minVersion, - }) + // For every patch, calculate the patches that it does not have. + utils.LogDebug("CHANGES VERSIONS", utils.LogFields{ + "Version": version, + "BaseVersion": change.BaseVersion, + "Diff": int(version - change.BaseVersion), + "Len": len(prevChangeStrs), + "ChangeStr": patchStr, + "minVersion": minVersion, + }) - //startIndex := len(prevChanges) - int(version-change.BaseVersion) - //if startIndex < 0 { - // utils.LogError("StartIndex is negative", ErrVersionOutOfDate, nil) - // return nil, -1, nil, ErrVersionOutOfDate - //} - - startIndex := int64(len(prevChanges) - 1) - - if change.BaseVersion > version { - // check to make sure the patch is being applied to the most recent revision - utils.LogError("BaseVersion too high", ErrVersionOutOfDate, nil) - return nil, -1, nil, 0, ErrVersionOutOfDate - } else if change.BaseVersion == version { - // If we are building on the server's base version, don't need to transform. - startIndex = int64(len(prevChanges)) - } else if change.BaseVersion < minVersion { - // if it's less than the minVersion, we've scrunched. - utils.LogError("BaseVersion less than minVersion", ErrVersionOutOfDate, nil) - return nil, -1, nil, 0, ErrVersionOutOfDate - } else if change.BaseVersion == minVersion { - // If it's equal to the minVersion, we use the entire array - startIndex = int64(0) - } else { - // Otherwise, find the right starting point - startIndex = int64(len(prevChanges)) - (version - change.BaseVersion) - for startIndex >= 0 && startIndex < int64(len(prevChanges)) { - otherPatch, err := patching.NewPatchFromString(prevChanges[startIndex]) - - if err != nil { - return nil, -1, nil, 0, ErrInternalServerError - } - - if change.BaseVersion > otherPatch.BaseVersion { - break - } else { - startIndex-- - } + //startIndex := len(prevChangeStrs) - int(version-change.BaseVersion) + //if startIndex < 0 { + // utils.LogError("StartIndex is negative", ErrVersionOutOfDate, nil) + // return nil, -1, nil, ErrVersionOutOfDate + //} + + startIndex := int64(len(prevChangeStrs) - 1) + + if change.BaseVersion > version { + // check to make sure the patch is being applied to the most recent revision + utils.LogError("BaseVersion too high", ErrVersionOutOfDate, nil) + return "", -1, nil, 0, ErrVersionOutOfDate + } else if change.BaseVersion == version { + // If we are building on the server's base version, don't need to transform. + startIndex = int64(len(prevChangeStrs)) + } else if change.BaseVersion < minVersion { + // if it's less than the minVersion, we've scrunched. + utils.LogError("BaseVersion less than minVersion", ErrVersionOutOfDate, nil) + return "", -1, nil, 0, ErrVersionOutOfDate + } else if change.BaseVersion == minVersion { + // If it's equal to the minVersion, we use the entire array + startIndex = int64(0) + } else { + // Otherwise, find the right starting point + startIndex = int64(len(prevChangeStrs)) - (version - change.BaseVersion) + for startIndex >= 0 && startIndex < int64(len(prevChangeStrs)) { + otherPatch, err := patching.NewPatchFromString(prevChangeStrs[startIndex]) + if err != nil { + utils.LogError("Failed to parse patch", err, utils.LogFields{ + "PatchStr": strings.Replace(prevChangeStrs[startIndex], "\n", "\\n", -1), + "StartIndex": startIndex, + }) + return "", -1, nil, 0, ErrInternalServerError } - startIndex++ // go back to the actual base version - } - // If it's negative at this point, it means we started off with an index that was less than -1. - // In other words, we've probably scrunched the changes we're looking for. - if startIndex < 0 { - utils.LogError("StartIndex was negative", ErrVersionOutOfDate, nil) - return nil, -1, nil, 0, ErrVersionOutOfDate + if change.BaseVersion > otherPatch.BaseVersion { + break + } else { + startIndex-- + } } + startIndex++ // go back to the actual base version + } - if startIndex < minStartIndex { - minStartIndex = startIndex - } + // If it's negative at this point, it means we started off with an index that was less than -1. + // In other words, we've probably scrunched the changes we're looking for. + if startIndex < 0 { + utils.LogError("StartIndex was negative", ErrVersionOutOfDate, nil) + return "", -1, nil, 0, ErrVersionOutOfDate + } - utils.LogDebug("FINISHED CHECKING", utils.LogFields{ - "Change": changeStr, - "StartIndex": startIndex, - "Len": len(prevChanges), - }) + if startIndex < minStartIndex { + minStartIndex = startIndex + } - // Apply patches from the change's baseVersion onwards - toApply := prevChanges[startIndex:] + utils.LogDebug("FINISHED CHECKING", utils.LogFields{ + "Change": patchStr, + "StartIndex": startIndex, + "Len": len(prevChangeStrs), + }) - utils.LogDebug("TRANSFORMING", utils.LogFields{ - "PatchesToApply": toApply, - "Change": changeStr, - "StartIndex": startIndex, - "Len": len(prevChanges), - }) + // Apply patches from the change's baseVersion onwards + toApply := prevChangeStrs[startIndex:] + + utils.LogDebug("TRANSFORMING", utils.LogFields{ + "PatchesToApply": toApply, + "Change": patchStr, + "StartIndex": startIndex, + "Len": len(prevChangeStrs), + }) - transformedPatch, err := change.TransformFromString(toApply, false) // rewrite change with transformed patch + transformedPatch := change + if startIndex != int64(len(prevChangeStrs)) { + consolidatedPatch, err := patching.ConsolidatePatches(prevChanges[startIndex:]) if err != nil { - return nil, -1, nil, 0, ErrInternalServerError // Could not parse one of the old patches - should never happen. + utils.LogError("Failed to consolidate patches", err, utils.LogFields{ + "Patch": strings.Replace(change.String(), "\n", "\\n", -1), + "prevChanges": strings.Replace(spew.Sprint(prevChanges), "\n", "\\n", -1), + }) } - // Transform against new patch; new patch has precedence. - // This allows multiple patches constructing a single block of text to stay together. (17/01/13) - for i := 0; i < len(toApply); i++ { - donePatch, err := patching.NewPatchFromString(toApply[i]) - if err != nil { - return nil, -1, nil, 0, ErrInternalServerError - } - - // Maintain donePatch's base version, so we don't mess anything up. - donePatchBaseVersion := donePatch.BaseVersion - donePatch = donePatch.Transform([]*patching.Patch{donePatch}, true) - donePatch.BaseVersion = donePatchBaseVersion - - toApply[i] = donePatch.String() + transformResults, err := patching.TransformPatches(change, consolidatedPatch) + if err != nil { + utils.LogError("Failed to transform patch", err, utils.LogFields{ + "Patch": strings.Replace(change.String(), "\n", "\\n", -1), + "consolidatedPatch": strings.Replace(consolidatedPatch.String(), "\n", "\\n", -1), + }) + return "", -1, nil, 0, err } - // Update the BaseVersion to be be the previous change - //transformedPatch.BaseVersion++ - transformedPatches = append(transformedPatches, transformedPatch.String()) + transformedPatch = transformResults.PatchXPrime + transformedPatch.BaseVersion = version } - /* - // THIS BLOCK OF CODE HAS BEEN DISABLED BECAUSE PATCH CONSOLIDATION DOES NOT WORK AS EXPECTED - // For this to correctly work, a new OT-like algorithm will need to be implemented. - - var consolidatedPatch *patching.Patch - for _, transformed := range transformedPatches { - if consolidatedPatch == nil { - consolidatedPatch = transformed - } else { - hoistedPatch := transformed.Transform([]*patching.Patch{consolidatedPatch.Undo()}) - newChanges := append(consolidatedPatch.Changes, hoistedPatch.Changes...) - sort.Sort(newChanges) - consolidatedPatch.Changes = newChanges - } - } - - // use the cas to make sure the document hasn't changed - builder := cb.bucket.MutateIn(key, cas, 0) - - if !useTemp { - builder.ArrayAppend("changes", consolidatedPatch.String(), false) - } else { - builder.ArrayAppend("tempchanges", consolidatedPatch.String(), false) - } - */ - // use the cas to make sure the document hasn't changed builder := cb.bucket.MutateIn(strconv.FormatInt(fileMeta.FileID, 10), gocb.Cas(cas), 0) if !useTemp { - builder.ArrayAppendMulti("changes", transformedPatches, false) + builder.ArrayAppendMulti("changes", []string{transformedPatch.String()}, false) } else { - builder.ArrayAppendMulti("tempchanges", transformedPatches, false) + builder.ArrayAppendMulti("tempchanges", []string{transformedPatch.String()}, false) } builder = builder.Counter("version", 1, false) _, err = builder.Execute() if err != nil { - return nil, -1, nil, 0, err + return "", -1, nil, 0, err } // TODO: Evaluate whether prevChangesCopy is the correct item to send back // use prevChangesCopy, so we don't send back the transformed patch set - return transformedPatches, version + 1, prevChangesCopy[minStartIndex:], len(prevChanges) + len(transformedPatches), err + return transformedPatch.String(), version + 1, prevChangesCopy[minStartIndex:], len(prevChangeStrs) + 1, err } diff --git a/modules/dbfs/couchbase_test.go b/modules/dbfs/couchbase_test.go index 69ce031..69ee4f7 100644 --- a/modules/dbfs/couchbase_test.go +++ b/modules/dbfs/couchbase_test.go @@ -185,10 +185,10 @@ func TestDatabaseImpl_CBAppendFileChange(t *testing.T) { di.CBDeleteFile(file.FileID) - patch1 := fmt.Sprintf("v%d:\n1:+6:patch1", originalFileVersion-1) - patch2 := fmt.Sprintf("v%d:\n2:+6:patch2", originalFileVersion-1) - patch3 := fmt.Sprintf("v%d:\n3:+6:patch3", originalFileVersion) - patch4 := fmt.Sprintf("v%d:\n4:+6:patch4", originalFileVersion) + patch1 := fmt.Sprintf("v%d:\n1:+6:patch1:\n4", originalFileVersion-1) + patch2 := fmt.Sprintf("v%d:\n2:+6:patch2:\n10", originalFileVersion-1) + patch3 := fmt.Sprintf("v%d:\n3:+6:patch3:\n4", originalFileVersion) + patch4 := fmt.Sprintf("v%d:\n4:+6:patch4:\n4", originalFileVersion) // although these are not valid patches, this is purely a test of the logic, not of the patching // because of that this might fail in the future @@ -199,7 +199,7 @@ func TestDatabaseImpl_CBAppendFileChange(t *testing.T) { changes, _, pulledVersion, _, err := di.PullChanges(file) assert.Equal(t, originalFileVersion, pulledVersion, "failed set up verification") - transformed, version, missing, lenChanges, err := di.CBAppendFileChange(file, []string{patch3}) + transformed, version, missing, lenChanges, err := di.CBAppendFileChange(file, patch3) assert.NoError(t, err, "unexpected error appending changes") assert.Empty(t, missing, "Unexpected missing patches") @@ -215,14 +215,13 @@ func TestDatabaseImpl_CBAppendFileChange(t *testing.T) { assert.Equal(t, patch1, changes[0], "first change was not correct") assert.Equal(t, patch2, changes[1], "second change was not correct") - assert.Len(t, transformed, 1, "returned unexpected number of transformed new changes") - assert.EqualValues(t, transformed[0], changes[2], "newly inserted change was not correct") + assert.EqualValues(t, transformed, changes[2], "newly inserted change was not correct") // Expect AppendFileChange to transform patch4, since it was based on the version created by patch2 changes, _, pulledVersion, _, err = di.PullChanges(file) assert.Equal(t, pulledVersion, version, "version pulled from the database does not match the one given when appending the change") - transformed, version, missing, lenChanges, err = di.CBAppendFileChange(file, []string{patch4}) + transformed, version, missing, lenChanges, err = di.CBAppendFileChange(file, patch4) assert.NoError(t, err, "unexpected error appending changes") assert.Len(t, missing, 1, "Unexpected number of missing patches") @@ -241,8 +240,7 @@ func TestDatabaseImpl_CBAppendFileChange(t *testing.T) { assert.Equal(t, patch2, changes[1], "second change was not correct") assert.Equal(t, patch3, changes[2], "third change was not correct") - assert.Len(t, transformed, 1, "returned unexpected number of transformed new changes") - assert.EqualValues(t, transformed[0], changes[3], "newly inserted change was not correct") + assert.EqualValues(t, transformed, changes[3], "newly inserted change was not correct") ver, err := di.CBGetFileVersion(file.FileID) assert.EqualValues(t, 4, ver, "wrong file version") diff --git a/modules/dbfs/databaseMock.go b/modules/dbfs/databaseMock.go index aeffca7..63cd3f8 100644 --- a/modules/dbfs/databaseMock.go +++ b/modules/dbfs/databaseMock.go @@ -135,27 +135,25 @@ func (dm *DatabaseMock) PullChanges(meta FileMeta) ([]string, uint64, int64, boo } // CBAppendFileChange is a mock of the real implementation -func (dm *DatabaseMock) CBAppendFileChange(file FileMeta, patches []string) ([]string, int64, []string, int, error) { +func (dm *DatabaseMock) CBAppendFileChange(file FileMeta, patch string) (string, int64, []string, int, error) { dm.FunctionCallCount++ - for _, changeStr := range patches { - change, err := patching.NewPatchFromString(changeStr) - if err != nil { - return nil, -1, nil, 0, errors.New("Failed to parse patch") - } + change, err := patching.NewPatchFromString(patch) + if err != nil { + return "", -1, nil, 0, errors.New("Failed to parse patch") + } - // check to make sure the patch is being applied to the most recent revision - if change.BaseVersion > dm.FileVersion[file.FileID] { - return nil, -1, nil, 0, ErrVersionOutOfDate - } + // check to make sure the patch is being applied to the most recent revision + if change.BaseVersion > dm.FileVersion[file.FileID] { + return "", -1, nil, 0, ErrVersionOutOfDate } dm.FileVersion[file.FileID]++ - newChanges := append(dm.FileChanges[file.FileID], patches...) + newChanges := append(dm.FileChanges[file.FileID], patch) dm.FileChanges[file.FileID] = newChanges - return patches, dm.FileVersion[file.FileID], nil, len(dm.FileChanges[file.FileID]), nil + return patch, dm.FileVersion[file.FileID], nil, len(dm.FileChanges[file.FileID]), nil } // mysql diff --git a/modules/dbfs/databases.go b/modules/dbfs/databases.go index 2577796..813d53a 100644 --- a/modules/dbfs/databases.go +++ b/modules/dbfs/databases.go @@ -1,8 +1,5 @@ package dbfs -// Dbfs is the globally used dbfs object for the server -var Dbfs DBFS - // DBFS is the interface which maps all of the necessary database and file system functions type DBFS interface { // multi @@ -43,7 +40,7 @@ type DBFS interface { // CBAppendFileChange mutates the file document with the new change and sets the new version number // Returns the new version number, the missing patches, the total count of patches tracked, and an error, if any. - CBAppendFileChange(file FileMeta, patches []string) ([]string, int64, []string, int, error) + CBAppendFileChange(file FileMeta, patches string) (string, int64, []string, int, error) // MySQL diff --git a/modules/dbfs/multi_test.go b/modules/dbfs/multi_test.go index 41b1c67..46cb56c 100644 --- a/modules/dbfs/multi_test.go +++ b/modules/dbfs/multi_test.go @@ -9,12 +9,14 @@ import ( "time" "github.com/CodeCollaborate/Server/modules/config" + "github.com/CodeCollaborate/Server/modules/patching" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) var defaultBaseFile = "this is a very important file" -var defaultChanges = []string{"v0:\n1:+5:test1", "v1:\n10:+5:test2"} -var transformedChanges = []string{"v0:\n1:+5:test1", "v1:\n10:+5:test2"} +var defaultChanges = []string{"v0:\n1:+5:test1:\n10", "v1:\n10:+5:test2:\n10"} +var transformedChanges = []string{"v0:\n1:+5:test1:\n10", "v1:\n10:+5:test2:\n10"} func setupFile(t *testing.T, baseFile string, baseChanges []string) (*DatabaseImpl, FileMeta) { testConfigSetup(t) @@ -36,7 +38,7 @@ func setupFile(t *testing.T, baseFile string, baseChanges []string) (*DatabaseIm assert.NoError(t, err, "error writing file to disk") for _, change := range baseChanges { - _, _, _, _, err = di.CBAppendFileChange(file, []string{change}) + _, _, _, _, err = di.CBAppendFileChange(file, change) assert.NoError(t, err, "error appending change to file") } @@ -64,14 +66,14 @@ func TestDatabaseImpl_ScrunchFile(t *testing.T) { for i := 0; i < 50; i++ { if i < 10 { - patches[i] = fmt.Sprintf("v%d:\n2:+1:%d", i, i) + patches[i] = fmt.Sprintf("v%d:\n2:+1:%d:\n10", i, i) } else { - patches[i] = fmt.Sprintf("v%d:\n2:+2:%d", i, i) + patches[i] = fmt.Sprintf("v%d:\n2:+2:%d:\n10", i, i) } } for i := 0; i < 5; i++ { - resultPatches[i] = fmt.Sprintf("v%d:\n2:+2:%d", i+45, i+45) + resultPatches[i] = fmt.Sprintf("v%d:\n2:+2:%d:\n10", i+45, i+45) } expectedOutput.WriteString("te") @@ -110,7 +112,7 @@ func TestDatabaseImpl_GetForScrunching(t *testing.T) { assert.NoError(t, err, "error getting swp or changes") assert.Len(t, changes, 1, "changes size was an unexpected length") - assert.Contains(t, changes, defaultChanges[0], "changes didn't contain correct change") + assert.Contains(t, changes, transformedChanges[0], "changes didn't contain correct change") assert.EqualValues(t, string(swp), string(defaultBaseFile), "swp file was not cloned properly") @@ -145,24 +147,28 @@ func TestDatabaseImpl_PullFile_MidDelete(t *testing.T) { defer os.RemoveAll(config.GetConfig().ServerConfig.ProjectPath) defer di.CBDeleteFile(file.FileID) - newChanges := []string{"v2:\n2:+1:2", "v2:\n2:+1:3", "v3:\n2:+1:4", "v4:\n2:+1:4", "v5:\n2:+1:5", "v6:\n2:+1:6", "v7:\n2:+1:7", "v8:\n2:+1:8", "v8:\n2:+1:9", "v8:\n2:+2:10"} - transformedNewChanges := []string{"v2:\n2:+1:2", "v2:\n2:+1:3", "v3:\n2:+1:4", "v4:\n2:+1:4", "v5:\n2:+1:5", "v6:\n2:+1:6", "v7:\n2:+1:7", "v8:\n2:+1:8", "v8:\n2:+1:9", "v8:\n2:+2:10"} + newChanges := []string{"v2:\n2:+1:2:\n10", "v2:\n2:+1:3:\n10", "v3:\n2:+1:4:\n10", "v4:\n2:+1:4:\n10", "v5:\n2:+1:5:\n10", "v6:\n2:+1:6:\n10", "v7:\n2:+1:7:\n10", "v8:\n2:+1:8:\n10", "v8:\n2:+1:9:\n10", "v8:\n2:+2:10:\n10"} + transformedNewChanges := []string{"v2:\n2:+2:32:\n10", "v3:\n2:+1:4:\n10", "v4:\n2:+1:4:\n10", "v5:\n2:+1:5:\n10", "v6:\n2:+1:6:\n10", "v7:\n2:+1:7:\n10", "v8:\n2:+4:1098:\n10"} newRawFile := []byte(string(defaultBaseFile) + "\nit's a pretty cool file, not going to lie\n") checkPullFile(t, di, file, transformedChanges, defaultBaseFile) // add more changes so it's more visible - appendChangeToFile(t, di, newChanges[:2]) + patches, err := patching.GetPatches(newChanges[:2]) + require.Nil(t, err) + patch, err := patching.ConsolidatePatches(patches) + require.Nil(t, err) + appendChangeToFile(t, di, patch.String()) - //checkPullFile(t, di, file, append(transformedChanges, transformedNewChanges[0]), defaultBaseFile) - checkPullFile(t, di, file, append(transformedChanges, transformedNewChanges[:2]...), defaultBaseFile) + checkPullFile(t, di, file, append(transformedChanges, transformedNewChanges[:1]...), defaultBaseFile) // arbitrarily saying we're going to scrunch off 2 patches - num := 2 + num := len(defaultChanges) + rem := 1 // make sure they're right //changes1, raw1, err := di.getForScrunching(file, 1) - changes1, raw1, err := di.getForScrunching(file, num) + changes1, raw1, err := di.getForScrunching(file, rem) assert.NoError(t, err, "error getting changes for scrunching") assert.EqualValues(t, string(defaultBaseFile), string(raw1), "swap was not made correctly") assert.Len(t, changes1, num, "pulled wrong number of changes") @@ -174,7 +180,7 @@ func TestDatabaseImpl_PullFile_MidDelete(t *testing.T) { // check pull file (expecting old + new changes w/ old base) //checkPullFile(t, di, file, append(transformedChanges, transformedNewChanges[:1]...), string(defaultBaseFile)) - checkPullFile(t, di, file, append(transformedChanges, transformedNewChanges[:2]...), string(defaultBaseFile)) + checkPullFile(t, di, file, append(transformedChanges, transformedNewChanges[:1]...), string(defaultBaseFile)) // START DELETE cb, err := di.openCouchBase() @@ -190,9 +196,9 @@ func TestDatabaseImpl_PullFile_MidDelete(t *testing.T) { nativeErr(t, err) // add change - appendChangeToFile(t, di, newChanges[2:3]) + appendChangeToFile(t, di, newChanges[2]) //checkPullFile(t, di, file, append(transformedChanges, transformedNewChanges[:2]...), string(defaultBaseFile)) - checkPullFile(t, di, file, append(transformedChanges, transformedNewChanges[:3]...), string(defaultBaseFile)) + checkPullFile(t, di, file, append(transformedChanges, transformedNewChanges[:2]...), string(defaultBaseFile)) // get changes in normal changes frag, err := cb.bucket.LookupIn(key).Get("changes").Execute() @@ -203,9 +209,9 @@ func TestDatabaseImpl_PullFile_MidDelete(t *testing.T) { nativeErr(t, err) // add change - appendChangeToFile(t, di, newChanges[3:4]) + appendChangeToFile(t, di, newChanges[3]) //checkPullFile(t, di, file, append(transformedChanges, transformedNewChanges[:3]...), string(defaultBaseFile)) - checkPullFile(t, di, file, append(transformedChanges, transformedNewChanges[:4]...), string(defaultBaseFile)) + checkPullFile(t, di, file, append(transformedChanges, transformedNewChanges[:3]...), string(defaultBaseFile)) // turn off writing to TempChanges & reset normal changes builder = cb.bucket.MutateIn(key, 0, 0) @@ -218,9 +224,9 @@ func TestDatabaseImpl_PullFile_MidDelete(t *testing.T) { // add change // check switched to swap - appendChangeToFile(t, di, newChanges[4:5]) + appendChangeToFile(t, di, newChanges[4]) //checkPullFile(t, di, file, transformedNewChanges[:4], string(newRawFile)) - checkPullFile(t, di, file, transformedNewChanges[:5], string(newRawFile)) + checkPullFile(t, di, file, transformedNewChanges[:4], string(newRawFile)) // get changes in TempChanges frag, err = cb.bucket.LookupIn(key).Get("tempchanges").Execute() @@ -232,17 +238,17 @@ func TestDatabaseImpl_PullFile_MidDelete(t *testing.T) { // add change // check switched to swap - appendChangeToFile(t, di, newChanges[5:6]) + appendChangeToFile(t, di, newChanges[5]) //checkPullFile(t, di, file, transformedNewChanges[:5], string(newRawFile)) - checkPullFile(t, di, file, transformedNewChanges[:6], string(newRawFile)) + checkPullFile(t, di, file, transformedNewChanges[:5], string(newRawFile)) err = di.swapSwp(file.RelativePath, file.Filename, file.ProjectID) assert.NoError(t, err, "Error swapping swap file, NOTE: the server WOULD normally be able to recover from here") // add change - appendChangeToFile(t, di, newChanges[6:7]) + appendChangeToFile(t, di, newChanges[6]) //checkPullFile(t, di, file, transformedNewChanges[:6], string(newRawFile)) - checkPullFile(t, di, file, transformedNewChanges[:7], string(newRawFile)) + checkPullFile(t, di, file, transformedNewChanges[:6], string(newRawFile)) // prepend changes and reset temporarily stored changes builder = cb.bucket.MutateIn(key, 0, 0) @@ -257,7 +263,11 @@ func TestDatabaseImpl_PullFile_MidDelete(t *testing.T) { assert.NoError(t, err, "Error deleting swap file, NOTE: the server WOULD normally be able to recover from here") // add change - appendChangeToFile(t, di, newChanges[7:]) + patches, err = patching.GetPatches(newChanges[7:]) + require.Nil(t, err) + patch, err = patching.ConsolidatePatches(patches) + require.Nil(t, err) + appendChangeToFile(t, di, patch.String()) checkPullFile(t, di, file, transformedNewChanges, string(newRawFile)) } @@ -265,8 +275,8 @@ func nativeErr(t *testing.T, err error) { assert.NoError(t, err, "error in naitive di.DeleteForScrunching code") } -func appendChangeToFile(t *testing.T, di *DatabaseImpl, changes []string) { - _, _, _, _, err := di.CBAppendFileChange(file, changes) +func appendChangeToFile(t *testing.T, di *DatabaseImpl, change string) { + _, _, _, _, err := di.CBAppendFileChange(file, change) assert.NoError(t, err, "Error while appending more changes") } diff --git a/modules/handlers/worker.go b/modules/handlers/worker.go new file mode 100644 index 0000000..714938c --- /dev/null +++ b/modules/handlers/worker.go @@ -0,0 +1,129 @@ +package handlers + +import ( + "errors" + + "github.com/CodeCollaborate/Server/modules/config" + "github.com/CodeCollaborate/Server/modules/datahandling" + "github.com/CodeCollaborate/Server/modules/dbfs" + "github.com/CodeCollaborate/Server/modules/rabbitmq" + "github.com/CodeCollaborate/Server/utils" +) + +// server only has one worker +var workerCfg *rabbitmq.AMQPPubSubCfg + +const workerName string = "datahandling_worker" +const workerOutboundQueueBufferSize int = 512 + +// StartWorker initializes the worker which talks with RabbitMQ for this server +func StartWorker(dbfsImpl dbfs.DBFS, prefetchCount int) *rabbitmq.AMQPPubSubCfg { + if workerCfg != nil { + // Worker already initialized, restarting + workerCfg.Control.Shutdown() + workerCfg = nil + } + + cfg := config.GetConfig() + + pubCfg := rabbitmq.NewPubConfig(func(msg rabbitmq.AMQPMessage) { + // do nothing (for now?) + msg.ErrHandler() + }, workerOutboundQueueBufferSize) + + subCfg := &rabbitmq.AMQPSubCfg{ + QueueName: workerName, + Keys: []string{}, + IsWorkQueue: true, + PrefetchCount: prefetchCount, + } + + workerCfg = rabbitmq.NewAMQPPubSubCfg(cfg.ServerConfig.Name, pubCfg, subCfg) + + subCfg.HandleMessageFunc = workerMessageHandler(dbfsImpl, workerCfg) + + go func() { + // auto-restart on failure + for !workerCfg.Control.HasExited() { + err := rabbitmq.RunPublisher(workerCfg) + if err != nil { + utils.LogError("Worker publisher error encountered. Exiting", err, nil) + } + } + }() + go func() { + // auto-restart on failure + for !workerCfg.Control.HasExited() { + err := rabbitmq.RunSubscriber(workerCfg) + if err != nil { + utils.LogError("Worker subscriber error encountered. Exiting", err, nil) + } + } + }() + + workerCfg.Control.Ready.Wait() + return workerCfg +} + +// WorkerEnqueue takes messages that need to be processed and sends them to RabbitMQ to be assigned to a worker +func WorkerEnqueue(message []byte, wsID uint64) error { + // can't naturally get wsID's of 0 + if wsID == 0 { + return errors.New("invalid websocketID given to worker") + } + + msg := rabbitmq.AMQPMessage{ + Headers: map[string]interface{}{ + "Origin": rabbitmq.LocalWebsocketName(wsID), + }, + RoutingKey: workerName, + ContentType: rabbitmq.ContentTypeWork, + Persistent: false, + Message: message, + } + + select { + case workerCfg.PubCfg.Messages <- msg: + default: + err := errors.New("Channel buffer full") + utils.LogError("Worker message queue full, failed to add new message", err, utils.LogFields{ + "AMQP Message": msg, + }) + return err + } + + return nil +} + +func workerMessageHandler(dbfsImpl dbfs.DBFS, cfg *rabbitmq.AMQPPubSubCfg) func(rabbitmq.AMQPMessage) error { + // have 1 dh per worker + dh := datahandling.DataHandler{ + MessageChan: cfg.PubCfg.Messages, + Db: dbfsImpl, + } + + return func(msg rabbitmq.AMQPMessage) error { + switch msg.ContentType { + case rabbitmq.ContentTypeWork: + // If notification with self as origin, early-out; ignore our own notifications. + wsOrigin, ok := msg.Headers["Origin"] + if !ok { + err := errors.New("Unnown message origin") + utils.LogError("Worker encountered ", err, utils.LogFields{ + "Message Headers": msg.Headers, // NOTE: message body could contain passwords + }) + return err + } + + go dh.Handle(msg.Message, wsOrigin.(string), msg.Ack) + return nil + default: + err := errors.New("Unnable to process RabbitMQ message type") + utils.LogError("not-work given to worker", err, utils.LogFields{ + "Message Headers": msg.Headers, + "Message Body": string(msg.Message), + }) + return err + } + } +} diff --git a/modules/handlers/wsmanager.go b/modules/handlers/wsmanager.go index 9f0c667..6a7ed85 100644 --- a/modules/handlers/wsmanager.go +++ b/modules/handlers/wsmanager.go @@ -3,16 +3,15 @@ package handlers import ( "errors" "net/http" - "sync" "sync/atomic" + "time" + + "github.com/gorilla/websocket" + "github.com/kr/pretty" "github.com/CodeCollaborate/Server/modules/config" - "github.com/CodeCollaborate/Server/modules/datahandling" - "github.com/CodeCollaborate/Server/modules/dbfs" "github.com/CodeCollaborate/Server/modules/rabbitmq" "github.com/CodeCollaborate/Server/utils" - "github.com/gorilla/websocket" - "github.com/kr/pretty" ) /** @@ -52,6 +51,8 @@ func NewWSConn(responseWriter http.ResponseWriter, request *http.Request) { defer wsConn.Close() cfg := config.GetConfig() + // TODO: Send data blob + // Generate unique ID for this websocket wsID := atomic.AddUint64(&atomicIDCounter, 1) @@ -60,15 +61,17 @@ func NewWSConn(responseWriter http.ResponseWriter, request *http.Request) { msg.ErrHandler() }, outboundMessageQueueBufferSize) + defer close(pubCfg.Messages) + subCfg := &rabbitmq.AMQPSubCfg{ - QueueID: wsID, + QueueName: rabbitmq.LocalWebsocketName(wsID), Keys: []string{}, IsWorkQueue: false, } pubSubCfg := rabbitmq.NewAMQPPubSubCfg(cfg.ServerConfig.Name, pubCfg, subCfg) - subCfg.HandleMessageFunc = newAMQPMessageHandler(wsID, pubSubCfg, wsConn) + subCfg.HandleMessageFunc = newClientMessageHandler(pubSubCfg, wsConn) go func() { err := rabbitmq.RunPublisher(pubSubCfg) @@ -87,43 +90,45 @@ func NewWSConn(responseWriter http.ResponseWriter, request *http.Request) { pubSubCfg.Control.Ready.Wait() - // we don't actually need more than 1 datahandler per websocket - dh := datahandling.DataHandler{ - MessageChan: pubCfg.Messages, - WebsocketID: wsID, - Db: dbfs.Dbfs, - } - - // Waitgroup to make sure channel is closed at appropriate time. - dhCompleted := &sync.WaitGroup{} - -loop: +msgLoop: for { select { case <-pubSubCfg.Control.Exit: - break loop + break msgLoop default: - messageType, message, err := wsConn.ReadMessage() + _, message, err := wsConn.ReadMessage() if err != nil { + if websocket.IsCloseError(err, websocket.CloseGoingAway, websocket.CloseMessage, websocket.CloseNoStatusReceived) { + break msgLoop + } utils.LogError("Failed to read message, terminating connection", err, nil) pubSubCfg.Control.Shutdown() - break loop + break msgLoop } - dhCompleted.Add(1) - go dh.Handle(messageType, message, dhCompleted) + err = WorkerEnqueue(message, wsID) + if err != nil { + utils.LogError("Failed to enqueue to worker, retrying", err, nil) + + time.Sleep(1 * time.Second) + err = WorkerEnqueue(message, wsID) + if err != nil { + utils.LogFatal("Failed to enqueue to worker", err, nil) + } + } } } - - // Wait for all datahandlers to complete before closing channel - dhCompleted.Wait() - close(pubCfg.Messages) } -func newAMQPMessageHandler(websocketID uint64, cfg *rabbitmq.AMQPPubSubCfg, wsConn *websocket.Conn) func(rabbitmq.AMQPMessage) error { - queueName := rabbitmq.RabbitWebsocketQueueName(websocketID) +func newClientMessageHandler(cfg *rabbitmq.AMQPPubSubCfg, wsConn *websocket.Conn) func(rabbitmq.AMQPMessage) error { + queueName := cfg.SubCfg.QueueName return func(msg rabbitmq.AMQPMessage) error { + err := msg.Ack() // ack early b/c regardless the outcome here we don't want to re-enqueue + utils.LogFatal("Error Ack'ing RabbitMQ message", err, utils.LogFields{ + "Message": string(msg.Message), + }) // only happens if we disconnect from rabbit + switch msg.ContentType { case rabbitmq.ContentTypeMsg: // If notification with self as origin, early-out; ignore our own notifications. @@ -141,11 +146,11 @@ func newAMQPMessageHandler(websocketID uint64, cfg *rabbitmq.AMQPPubSubCfg, wsCo rch := rabbitmq.RabbitCommandHandler{ ExchangeName: cfg.ExchangeName, WSConn: wsConn, - WSID: cfg.SubCfg.QueueID, + QueueName: cfg.SubCfg.QueueName, } return rch.HandleCommand(msg) default: - err := errors.New("No such ContentType") + err := errors.New("Unnable to process RabbitMQ message type") utils.LogError("Invalid ContentType", err, utils.LogFields{ "AMQPMessage": pretty.Sprint(msg), }) diff --git a/modules/patching/consolidation.go b/modules/patching/consolidation.go new file mode 100644 index 0000000..f6feea5 --- /dev/null +++ b/modules/patching/consolidation.go @@ -0,0 +1,217 @@ +package patching + +import ( + "errors" +) + +// ConsolidatePatches consolidates patch others with patch A. +// Patches should be fed into this function in dependency order (A -> B -> C) +func ConsolidatePatches(patches []*Patch) (*Patch, error) { + if len(patches) <= 0 { + return nil, errors.New("ConsolidatePatches: No patches provided") + } + + patchA := patches[0] + // Consolidate first two patches, if there are any + for _, patchB := range patches[1:] { + indexA := -1 + indexB := -1 + resultDiffs := Diffs{} + currIndex := 0 + + diffA, indexA := getNextDiff(patchA, indexA, false) + diffB, indexB := getNextDiff(patchB, indexB, false) + + // Convenience update functions + getNextDiffA := func() { + diffA, indexA = getNextDiff(patchA, indexA, isNoOp(diffA)) + } + getNextDiffB := func() { + diffB, indexB = getNextDiff(patchB, indexB, isNoOp(diffB)) + } + commit := func(diff *Diff, numChars int) { + if numChars == -1 { + resultDiffs = append(resultDiffs, NewDiff(diff.Insertion, currIndex, diff.Changes)) + } else { + resultDiffs = append(resultDiffs, NewDiff(diff.Insertion, currIndex, diff.Changes[:numChars])) + } + } + + // If either diff is nil, we have hit the end + for diffA != nil && diffB != nil { + // Get lengths of each diff + lenA := 0 + lenB := 0 + if isNoOp(diffA) { + lenA = noOpLength(diffA, patchA, indexA) + } else { + lenA = diffA.Length() + } + if isNoOp(diffB) { + lenB = noOpLength(diffB, patchB, indexB) + } else { + lenB = diffB.Length() + } + + if !diffA.Insertion && !isNoOp(diffA) { // Cases A1 through A3 + commit(diffA, -1) + currIndex += lenA + getNextDiffA() + } else if diffB.Insertion && !isNoOp(diffB) { // Cases G1 through G3 + commit(diffB, -1) + getNextDiffB() + } else { + // Commit changes and update currIndex as needed + switch { + // Cases B1 through B3 + case !isNoOp(diffA) && diffA.Insertion && !isNoOp(diffB) && !diffB.Insertion: + // Commit type 0 + default: + // Do nothing + break + + // Cases C1 through C3 + case !isNoOp(diffA) && diffA.Insertion && isNoOp(diffB): + switch { + // Commit type 1 (Cases C1, C2) + case lenA < lenB, lenA == lenB: + commit(diffA, -1) + + // Commit type 2 (Case C3) + default: + commit(diffA, lenB) + } + + // Cases H1 through H3 + case isNoOp(diffA) && !isNoOp(diffB) && !diffB.Insertion: + switch { + // Commit Type 3 (Case H1) + case lenA < lenB: + commit(diffB, lenA) + currIndex += lenA + // Commit Type 4 (Case H2) + case lenA == lenB: + commit(diffB, -1) + currIndex += lenA + // Commit Type 5 (Case H3) + default: + commit(diffB, -1) + currIndex += lenB + } + // Case I1 through I3 + case isNoOp(diffA) && isNoOp(diffB): + switch { + // Commit type 6 (Cases I1, I2) + case lenA < lenB, lenA == lenB: + currIndex += lenA + // Commit Type 7 (Cases I3) + default: + currIndex += lenB + } + } + + // Update diff, get a new one if needed. + switch { + // Iteration Type 1 (All cases in 1 column) + case lenA < lenB: + if isNoOp(diffB) { + diffB.StartIndex += lenA + } else { + diffB.Changes = diffB.Changes[lenA:] + } + getNextDiffA() + + // Iteration Type 2 (All cases in 2 column) + case lenA == lenB: + getNextDiffA() + getNextDiffB() + + // Iteration Type 3 (All cases in 3 column) + default: + if isNoOp(diffA) { + diffA.StartIndex += lenB + } else { + diffA.Changes = diffA.Changes[lenB:] + } + getNextDiffB() + } + } + } + patchA = NewPatch(patchA.BaseVersion, resultDiffs, patchA.DocLength) + } + return patchA, nil +} + +func getNextDiff(patch *Patch, currIndex int, wasNoOp bool) (*Diff, int) { + // If we are just starting, currIndex will be -1 + if currIndex == -1 { + if !wasNoOp { // If we are just starting, wasNoOp will be false - return the starting no-op + return NewDiff(true, 0, ""), -1 + } + + // If the length of the patches is 0, return nil diff + if patch.Changes.Len() <= 0 { + return nil, -1 + } + + // Else return the first diff + return patch.Changes[0].clone(), 0 + } + + // If previous one was a noOp, return either the end or the next actual diff + if wasNoOp { + // Return nil diff if we have gone past the end + if currIndex+1 >= patch.Changes.Len() { + return nil, -1 + } + // Return next diff otherwise. + return patch.Changes[currIndex+1].clone(), currIndex + 1 + } + + // Else return the next value + currDiff := patch.Changes[currIndex] + // If we have no more slice, and our current diff does not go to the end, return a new noop diff + if currIndex+1 >= patch.Changes.Len() { + // If it is an insertion, return new noOp diff at start index. + if currDiff.Insertion { + return NewDiff(true, currDiff.StartIndex, ""), currIndex + } + + // Else, return a noop diff after the removed block + return NewDiff(true, currDiff.StartIndex+currDiff.Length(), ""), currIndex + + } + + //return the next diff if it is adjacent, or a noOp otherwise. + nextDiff := patch.Changes[currIndex+1] + + // If next diff is adjacent, return it directly + if nextDiff.StartIndex == currDiff.StartIndex || !currDiff.Insertion && currDiff.StartIndex+currDiff.Length() >= nextDiff.StartIndex { + return nextDiff.clone(), currIndex + 1 + } + + // Otherwise, return a new no-op diff + // If the current diff is an insertion, start the no-op diff at the current location + if currDiff.Insertion { + return NewDiff(true, currDiff.StartIndex, ""), currIndex + } + + // Else, start it at the end of the removal + return NewDiff(true, currDiff.StartIndex+currDiff.Length(), ""), currIndex + +} + +func isNoOp(diff *Diff) bool { + return diff.Length() == 0 +} + +func noOpLength(diff *Diff, patch *Patch, currIndex int) int { + // If this is the last diff, return the remaining untouched length of the document + if currIndex+1 >= patch.Changes.Len() { + return patch.DocLength - diff.StartIndex + 1 + } + + // Else, return the length from the no-op patch's startIndex to the next patch's startIndex + return patch.Changes[currIndex+1].StartIndex - diff.StartIndex + +} diff --git a/modules/patching/consolidation_test.go b/modules/patching/consolidation_test.go new file mode 100644 index 0000000..3e1ba55 --- /dev/null +++ b/modules/patching/consolidation_test.go @@ -0,0 +1,528 @@ +package patching + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetNextDiff(t *testing.T) { + tests := []struct { + desc string + patch *Patch + currIndex int + wasNoOp bool + expectedDiff string + expectedIndex int + }{ + { + desc: "Starting noOp", + patch: getPatchesOrDie(t, "v0:\n1:+1:A:\n5")[0], + currIndex: -1, + wasNoOp: false, + expectedDiff: "0:+0:", + expectedIndex: -1, + }, + { + desc: "First patch after noOp", + patch: getPatchesOrDie(t, "v0:\n1:+1:A:\n5")[0], + currIndex: -1, + wasNoOp: true, + expectedDiff: "1:+1:A", + expectedIndex: 0, + }, + { + desc: "Ending noOp", + patch: getPatchesOrDie(t, "v0:\n1:+1:A:\n5")[0], + currIndex: 0, + wasNoOp: false, + expectedDiff: "1:+0:", + expectedIndex: 0, + }, + { + desc: "Insertion noOp", + patch: getPatchesOrDie(t, "v0:\n1:+1:A:\n5")[0], + currIndex: 0, + wasNoOp: false, + expectedDiff: "1:+0:", + expectedIndex: 0, + }, + { + desc: "Removal noOp", + patch: getPatchesOrDie(t, "v0:\n1:-1:A:\n5")[0], + currIndex: 0, + wasNoOp: false, + expectedDiff: "2:+0:", + expectedIndex: 0, + }, + { + desc: "Beyond ending noOp", + patch: getPatchesOrDie(t, "v0:\n1:+1:A:\n5")[0], + currIndex: 0, + wasNoOp: true, + expectedDiff: "nil", + expectedIndex: -1, + }, + { + desc: "Starting noOp with first diff at 0", + patch: getPatchesOrDie(t, "v0:\n0:+1:A:\n5")[0], + currIndex: -1, + wasNoOp: false, + expectedDiff: "0:+0:", + expectedIndex: -1, + }, + { + desc: "Ending noOp after noOp with first diff at 0", + patch: getPatchesOrDie(t, "v0:\n0:+1:A:\n5")[0], + currIndex: 0, + wasNoOp: false, + expectedDiff: "0:+0:", + expectedIndex: 0, + }, + { + desc: "Ending noOp with no space after", + patch: getPatchesOrDie(t, "v0:\n0:+1:A:\n0")[0], + currIndex: 0, + wasNoOp: false, + expectedDiff: "0:+0:", + expectedIndex: 0, + }, + { + desc: "Beyond ending noOp", + patch: getPatchesOrDie(t, "v0:\n0:+1:A:\n5")[0], + currIndex: 0, + wasNoOp: true, + expectedDiff: "nil", + expectedIndex: -1, + }, + { + desc: "Adjacent patch insert-insert 1", + patch: getPatchesOrDie(t, "v0:\n0:+1:A,\n1:+1:B:\n5")[0], + currIndex: 0, + wasNoOp: false, + expectedDiff: "0:+0:", + expectedIndex: 0, + }, + { + desc: "Adjacent patch insert-insert 2", + patch: getPatchesOrDie(t, "v0:\n0:+1:A,\n1:+1:B:\n5")[0], + currIndex: 0, + wasNoOp: true, + expectedDiff: "1:+1:B", + expectedIndex: 1, + }, + { + desc: "Adjacent patch insert-remove 1", + patch: getPatchesOrDie(t, "v0:\n0:+1:A,\n1:-1:B:\n5")[0], + currIndex: 0, + wasNoOp: false, + expectedDiff: "0:+0:", + expectedIndex: 0, + }, + { + desc: "Adjacent patch insert-remove 2", + patch: getPatchesOrDie(t, "v0:\n0:+1:A,\n1:-1:B:\n5")[0], + currIndex: 0, + wasNoOp: true, + expectedDiff: "1:-1:B", + expectedIndex: 1, + }, + { + desc: "Adjacent patch remove-insert", + patch: getPatchesOrDie(t, "v0:\n0:-1:A,\n1:+1:B:\n5")[0], + currIndex: 0, + wasNoOp: false, + expectedDiff: "1:+1:B", + expectedIndex: 1, + }, + } + + for _, test := range tests { + diff, index := getNextDiff(test.patch, test.currIndex, test.wasNoOp) + + if test.expectedDiff == "nil" { + assert.Nil(t, diff, "TestGetNextDiff[%s]: Unexpected nil diff", test.desc) + } else { + assert.Equal(t, test.expectedDiff, diff.String(), "TestGetNextDiff[%s]: Unexpected diff", test.desc) + } + assert.Equal(t, test.expectedIndex, index, "TestGetNextDiff[%s]: Unexpected index", test.desc) + } +} + +type overallConsolidationTest struct { + desc string + baseText string + patches []*Patch + error error +} + +func TestConsolidatePatch(t *testing.T) { + tests := []overallConsolidationTest{ + { + desc: "Simple Add-Only test", + baseText: "", + patches: getPatchesOrDie(t, "v0:\n0:+7:testing:\n0", "v1:\n1:+2:AB,\n4:+4:CDEF,\n7:+3:GHI:\n7"), + }, + { + desc: "Simple deletion-addition test 1", + baseText: "testing", + patches: getPatchesOrDie(t, "v0:\n2:-2:st,\n5:-1:n:\n7", "v1:\n1:+2:AB,\n4:+4:CDEF:\n4"), + }, + { + desc: "Simple deletion-addition test 2", + baseText: "testing", + patches: getPatchesOrDie(t, "v0:\n2:-2:st,\n5:-1:n:\n7", "v1:\n0:-2:te,\n4:+4:CDEF:\n4"), + }, + { + desc: "Mixed deletion-addition test 1", + baseText: "testing", + patches: getPatchesOrDie(t, "v0:\n1:+2:AB,\n5:-1:n:\n7", "v1:\n0:-2:tA,\n4:+4:CDEF:\n8"), + }, + { + desc: "Mixed deletion-addition test 2", + baseText: "testing", + patches: getPatchesOrDie(t, "v0:\n1:-3:est,\n5:+2:AB:\n6", "v1:\n0:-2:ti,\n3:+4:CDEF:\n9"), + }, + } + + for _, test := range tests { + patchedText, err := PatchText(test.baseText, test.patches) + require.Nil(t, err) + + consolidatedPatch, err := ConsolidatePatches(test.patches) + require.Nil(t, err) + + consolidatedPatchedText, err := PatchText(test.baseText, []*Patch{consolidatedPatch}) + require.Equal(t, patchedText, consolidatedPatchedText, "TestConsolidatePatch[%s]: Expected %s but got %s", test.desc, patchedText, consolidatedPatchedText) + } +} + +func TestConsolidatePatchLong(t *testing.T) { + tests := []overallConsolidationTest{ + { + desc: "File construction 1", + baseText: "", + patches: getPatchesOrDie(t, + "v1:\n0:+1:y:\n0", "v2:\n1:+1:e:\n1", "v3:\n2:+1:p:\n2", "v4:\n3:+1:%0A:\n3", "v5:\n4:+1:%0A:\n4", + "v6:\n3:+1:%0A:\n5", "v7:\n6:+1:O:\n6", "v8:\n4:+1:%0A:\n7", "v9:\n5:+1:%0A:\n8", "v9:\n6:+1:%0A:\n9", + "v10:\n9:-1:O:\n10", "v11:\n6:+1:I:\n9", "v12:\n7:+1:%27:\n10", "v13:\n11:+1:O:\n11", "v14:\n12:+1:k:\n12", + "v15:\n8:+1:m:\n13", "v16:\n9:+1:+:\n14", "v16:\n10:+1:n:\n15", "v17:\n11:+1:o:\n16", "v17:\n12:+1:t:\n17", + "v17:\n13:+1:+:\n18", "v17:\n14:+1:s:\n19", "v17:\n15:+1:u:\n20", "v18:\n16:+1:p:\n21", "v19:\n17:+1:e:\n22", + "v20:\n18:+1:r:\n23", "v21:\n19:+1:+:\n24", "v21:\n20:+1:g:\n25", "v21:\n21:+1:o:\n26", "v21:\n22:+1:o:\n27", + "v21:\n23:+1:d:\n28", "v21:\n24:+1:+:\n29", "v22:\n25:+1:a:\n30", "v22:\n26:+1:t:\n31", "v23:\n27:+1:+:\n32", + "v24:\n33:+1:.:\n33", "v25:\n28:+1:t:\n34", "v26:\n29:+1:y:\n35", "v27:\n30:+1:p:\n36", "v28:\n31:+1:i:\n37", + "v29:\n32:+1:n:\n38", "v30:\n33:+1:g:\n39", "v31:\n34:+1:+:\n40", "v32:\n35:+1:h:\n41", "v33:\n36:+1:o:\n42", + "v34:\n37:+1:n:\n43", "v35:\n38:+1:e:\n44", "v36:\n39:+1:s:\n45", "v37:\n46:+1:+:\n46", "v37:\n47:+1:S:\n47", + "v37:\n48:+1:o:\n48", "v37:\n49:+1:+:\n49", "v38:\n40:+1:t:\n50", "v38:\n41:+1:l:\n51", "v38:\n42:+1:y:\n52", + "v39:\n53:+1:w:\n53", "v39:\n54:+1:e:\n54", "v39:\n54:-1:e:\n55", "v39:\n53:-1:w:\n54", "v39:\n53:+1:w:\n53", + "v39:\n54:+1:h:\n54", "v39:\n55:+1:e:\n55", "v39:\n56:+1:n:\n56", "v39:\n57:+1:+:\n57", "v39:\n58:+1:w:\n58", + "v39:\n59:+1:e:\n59", "v39:\n60:+1:+:\n60", "v39:\n61:+1:a:\n61", "v39:\n62:+1:r:\n62", "v39:\n63:+1:e:\n63", + "v39:\n64:+1:+:\n64", "v39:\n65:+1:t:\n65", "v39:\n66:+1:y:\n66", "v39:\n67:+1:p:\n67", "v39:\n68:+1:i:\n68", + "v39:\n69:+1:n:\n69", "v39:\n70:+1:g:\n70", "v39:\n71:+1:+:\n71", "v39:\n72:+1:t:\n72", "v39:\n73:+1:o:\n73", + "v39:\n74:+1:g:\n74", "v39:\n75:+1:e:\n75", "v40:\n76:+1:t:\n76", "v40:\n77:+1:h:\n77", "v41:\n78:+1:e:\n78", + "v42:\n79:+1:r:\n79", "v43:\n80:+1:%2C:\n80", "v43:\n81:+1:+:\n81", "v43:\n82:+1:i:\n82", "v44:\n83:+1:t:\n83", + "v45:\n84:+1:+:\n84", "v46:\n85:+1:u:\n85", "v47:\n86:+1:p:\n86", "v48:\n87:+1:d:\n87", "v49:\n43:+1:.:\n88", + "v50:\n44:+1:.:\n89", "v51:\n45:+1:.:\n90", "v52:\n91:+1:a:\n91", "v53:\n92:+1:t:\n92", "v53:\n93:+1:e:\n93", + "v53:\n94:+1:s:\n94", "v53:\n95:+1:%2C:\n95", "v53:\n96:+1:+:\n96", "v53:\n97:+1:a:\n97", "v53:\n98:+1:n:\n98", + "v53:\n99:+1:d:\n99", "v53:\n100:+1:+:\n100", "v53:\n101:+1:d:\n101", "v53:\n102:+1:o:\n102", "v53:\n103:+1:e:\n103", + "v53:\n104:+1:s:\n104", "v53:\n105:+1:+:\n105", "v53:\n106:+1:i:\n106", "v53:\n107:+1:t:\n107", "v53:\n108:+1:+:\n108", + "v53:\n109:+1:i:\n109", "v53:\n110:+1:n:\n110", "v53:\n111:+1:+:\n111", "v53:\n112:+1:t:\n112", "v53:\n113:+1:h:\n113", + "v53:\n114:+1:e:\n114", "v53:\n115:+1:+:\n115", "v53:\n116:+1:c:\n116", "v53:\n117:+1:o:\n117", "v54:\n118:+1:r:\n118", + "v54:\n119:+1:r:\n119", "v54:\n120:+1:e:\n120", "v55:\n121:+1:c:\n121", "v56:\n122:+1:+:\n122", "v56:\n123:+1:t:\n123", + "v57:\n46:+1:%0A:\n124", "v58:\n47:+1:%0A:\n125", "v58:\n48:+1:s:\n126", "v58:\n49:+1:o:\n127", "v58:\n50:+1:m:\n128", + "v58:\n51:+1:e:\n129", "v58:\n52:+1:t:\n130", "v58:\n53:+1:i:\n131", "v58:\n54:+1:m:\n132", "v58:\n55:+1:e:\n133", + "v58:\n56:+1:s:\n134", "v58:\n57:+1:+:\n135", "v59:\n135:-1:t:\n136", "v60:\n134:-1:+:\n135", "v60:\n134:+1:t:\n134", + "v60:\n135:+1:+:\n135", "v60:\n136:+1:l:\n136", "v61:\n137:+1:o:\n137", "v61:\n138:+1:c:\n138", "v61:\n139:+1:a:\n139", + "v61:\n140:+1:t:\n140", "v61:\n141:+1:i:\n141", "v61:\n142:+1:o:\n142", "v61:\n143:+1:n:\n143", "v61:\n144:+1:.:\n144", + "v62:\n58:+1:i:\n145", "v63:\n146:+1:%0A:\n146", "v64:\n147:+1:%0A:\n147", "v65:\n148:+1:S:\n148", "v66:\n149:+1:o:\n149", + "v67:\n59:+1:t:\n150", "v67:\n60:+1:+:\n151", "v68:\n152:+1:+:\n152", "v69:\n61:+1:w:\n153", "v69:\n62:+1:i:\n154", + "v69:\n63:+1:l:\n155", "v69:\n64:+1:l:\n156", "v69:\n65:+1:+:\n157", "v69:\n66:+1:k:\n158", "v69:\n67:+1:i:\n159", + "v69:\n68:+1:n:\n160", "v69:\n69:+1:d:\n161", "v69:\n70:+1:a:\n162", "v70:\n71:+1:+:\n163", "v70:\n72:+1:w:\n164", + "v71:\n72:-1:w:\n165", "v72:\n72:+1:t:\n164", "v72:\n73:+1:a:\n165", "v72:\n74:+1:k:\n166", "v72:\n75:+1:e:\n167", + "v72:\n76:+1:+:\n168", "v72:\n77:+1:a:\n169", "v72:\n78:+1:+:\n170", "v72:\n79:+1:l:\n171", "v72:\n80:+1:i:\n172", + "v72:\n81:+1:t:\n173", "v72:\n82:+1:t:\n174", "v72:\n83:+1:l:\n175", "v72:\n84:+1:e:\n176", "v72:\n85:+1:+:\n177", + "v72:\n86:+1:b:\n178", "v72:\n87:+1:i:\n179", "v72:\n88:+1:t:\n180", "v73:\n89:+1:+:\n181", "v73:\n90:+1:t:\n182", + "v73:\n91:+1:o:\n183", "v73:\n92:+1:+:\n184", "v73:\n93:+1:c:\n185", "v73:\n94:+1:a:\n186", "v74:\n95:+1:t:\n187", + "v75:\n96:+1:c:\n188", "v76:\n97:+1:h:\n189", "v77:\n189:-1:+:\n190", "v78:\n188:-1:o:\n189", "v79:\n187:-1:S:\n188", + "v80:\n187:+1:I:\n187", "v81:\n188:+1:+:\n188", "v82:\n189:+1:s:\n189", "v83:\n190:+1:u:\n190", "v84:\n191:+1:s:\n191", + "v85:\n192:+1:p:\n192", "v85:\n193:+1:e:\n193", "v85:\n194:+1:c:\n194", "v86:\n195:+1:t:\n195", "v87:\n196:+1:+:\n196", + "v88:\n197:+1:t:\n197", "v89:\n198:+1:h:\n198", "v89:\n199:+1:a:\n199", "v89:\n200:+1:t:\n200", "v89:\n201:+1:+:\n201", + "v89:\n202:+1:i:\n202", "v89:\n203:+1:s:\n203", "v89:\n204:+1:+:\n204", "v89:\n205:+1:p:\n205", "v90:\n206:+1:i:\n206", + "v90:\n207:+1:n:\n207", "v90:\n208:+1:g:\n208", "v90:\n209:+1:.:\n209", "v91:\n98:+1:+:\n210", "v91:\n99:+1:u:\n211", + "v91:\n100:+1:p:\n212", "v91:\n101:+1:+:\n213", "v92:\n101:-1:+:\n214", "v93:\n213:+1:%0A:\n213", "v94:\n214:+1:%0A:\n214", + "v94:\n215:+1:%0A:\n215", + "v95:\n0:-216:yep%0A%0A%0AI%27m+not+super+good+at+typing+honestly...%0A%0Asometimes+it+will+kinda+take+a+little+bit+to+catch+up%0A%0A%0AOk.+So+when+we+are+typing+together%2C+it+updates%2C+and+does+it+in+the+correct+location.%0A%0AI+suspect+that+is+ping.%0A%0A%0A:\n216", + "v96:\n0:+1:t:\n0", "v97:\n1:+1:e:\n1", "v98:\n2:+1:s:\n2", "v99:\n3:+1:t:\n3", "v100:\n4:+1:i:\n4", + "v101:\n5:+1:n:\n5", "v102:\n6:+1:g:\n6", "v103:\n6:-1:g:\n7", "v104:\n5:-1:n:\n6", "v105:\n4:-1:i:\n5", + "v106:\n3:-1:t:\n4", "v107:\n2:-1:s:\n3", "v108:\n1:-1:e:\n2", "v109:\n0:-1:t:\n1", "v110:\n0:+1:l:\n0", + "v111:\n1:+1:t:\n1", "v112:\n2:+1:j:\n2", "v113:\n3:+1:s:\n3", "v114:\n4:+1:l:\n4", "v115:\n5:+1:e:\n5", + "v116:\n6:+1:k:\n6", "v117:\n7:+1:l:\n7", "v118:\n8:+1:a:\n8", "v119:\n9:+1:s:\n9", "v120:\n10:+1:d:\n10", + "v121:\n11:+1:f:\n11", "v122:\n0:-12:ltjsleklasdf,\n0:+1:H:\n12", "v123:\n1:+1:e:\n1", "v124:\n2:+1:l:\n2", + "v125:\n3:+1:l:\n3", "v126:\n4:+1:o:\n4", "v127:\n5:+1:+:\n5", "v128:\n6:+1:m:\n6", "v129:\n7:+1:y:\n7", + "v130:\n8:+1:+:\n8", "v131:\n9:+1:n:\n9", "v132:\n10:+1:a:\n10", "v133:\n11:+1:m:\n11", "v134:\n12:+1:e:\n12", + "v135:\n13:+1:+:\n13", "v136:\n14:+1:i:\n14", "v137:\n15:+1:s:\n15", "v138:\n16:+1:+:\n16", "v139:\n17:+1:b:\n17", + "v140:\n18:+1:e:\n18", "v141:\n19:+1:n:\n19", "v142:\n20:+1:.:\n20", "v143:\n21:+1:+:\n21", "v144:\n22:+1:%0A:\n22", + "v145:\n23:+1:%0A:\n23", "v146:\n24:+1:I:\n24", "v147:\n25:+1:t:\n25", "v148:\n26:+1:+:\n26", "v149:\n27:+1:w:\n27", + "v150:\n27:-1:w:\n28", "v151:\n27:+1:h:\n27", "v152:\n28:+1:a:\n28", "v153:\n29:+1:n:\n29", "v154:\n30:+1:d:\n30", + "v155:\n31:+1:l:\n31", "v156:\n32:+1:e:\n32", "v157:\n33:+1:s:\n33", "v158:\n34:+1:c:\n34", "v159:\n35:+1:o:\n35", + "v160:\n36:+1:n:\n36", "v161:\n37:+1:c:\n37", "v162:\n38:+1:u:\n38", "v163:\n39:+1:r:\n39", "v164:\n40:+1:r:\n40", + "v165:\n41:+1:e:\n41", "v166:\n42:+1:n:\n42", "v167:\n43:+1:t:\n43", "v168:\n44:+1:+:\n44", "v169:\n45:+1:e:\n45", + "v170:\n46:+1:d:\n46", "v171:\n47:+1:i:\n47", "v172:\n48:+1:t:\n48", "v173:\n49:+1:s:\n49", "v174:\n50:+1:+:\n50", + "v175:\n51:+1:o:\n51", "v176:\n52:+1:n:\n52", "v177:\n53:+1:+:\n53", "v178:\n0:+1:d:\n54", "v179:\n1:+1:i:\n55", + "v180:\n2:+1:f:\n56", "v181:\n3:+1:f:\n57", "v182:\n4:+1:e:\n58", "v183:\n5:+1:r:\n59", "v184:\n6:+1:e:\n60", + "v185:\n7:+1:n:\n61", "v186:\n8:+1:t:\n62", "v187:\n9:+1:+:\n63", "v188:\n10:+1:e:\n64", "v189:\n11:+1:d:\n65", + "v190:\n12:+1:i:\n66", "v191:\n13:+1:t:\n67", "v192:\n14:+1:o:\n68", "v193:\n15:+1:r:\n69", "v194:\n16:+1:s:\n70", + "v195:\n17:+1:+:\n71", "v196:\n72:+1:p:\n72", "v197:\n73:+1:e:\n73", "v198:\n74:+1:r:\n74", "v199:\n75:+1:f:\n75", + "v200:\n76:+1:e:\n76", "v201:\n77:+1:c:\n77", "v202:\n78:+1:t:\n78", "v203:\n79:+1:+:\n79", "v204:\n80:+1:l:\n80", + "v205:\n81:+1:y:\n81", "v206:\n82:+1:+:\n82", "v207:\n82:-1:+:\n83", "v208:\n81:-1:y:\n82", "v209:\n80:-1:l:\n81", + "v210:\n79:-1:+:\n80", "v211:\n79:+1:l:\n79", "v212:\n80:+1:y:\n80", "v213:\n81:+1:+:\n81", "v214:\n82:+1:w:\n82", + "v215:\n83:+1:e:\n83", "v216:\n84:+1:l:\n84", "v217:\n85:+1:l:\n85", "v218:\n86:+1:.:\n86", + "v219:\n0:-87:different+editors+Hello+my+name+is+ben.+%0A%0AIt+handlesconcurrent+edits+on+perfectly+well.,\n0:+1:t:\n87", + "v220:\n1:+1:e:\n1", "v221:\n2:+1:s:\n2", "v222:\n3:+1:t:\n3", "v223:\n4:+1:i:\n4", "v224:\n5:+1:n:\n5", "v225:\n6:+1:g:\n6", + "v226:\n7:+1:+:\n7", "v227:\n8:+1:h:\n8", "v228:\n9:+1:e:\n9", "v229:\n10:+1:l:\n10", "v230:\n11:+1:l:\n11", ""+"v231:\n12:+1:o:\n12", + "v232:\n13:+1:e:\n13", "v233:\n14:+1:s:\n14", "v234:\n15:+1:a:\n15", "v235:\n16:+1:f:\n16", + ), + }, + { + desc: "File construction 2", + baseText: "", + patches: getPatchesOrDie(t, + "v1:\n0:+1:t:\n0", "v2:\n1:+1:e:\n1", "v2:\n2:+1:s:\n2", "v2:\n3:+1:t:\n3", "v3:\n4:+1:t:\n4", + "v4:\n4:-1:t:\n5", "v5:\n3:-1:t:\n4", "v6:\n2:-1:s:\n3", "v7:\n1:-1:e:\n2", "v8:\n0:-1:t:\n1", + "v9:\n0:+1:k:\n0", "v10:\n0:+1:j:\n1", "v11:\n2:+1:e:\n2", "v12:\n2:-1:e:\n3", "v13:\n1:-1:k:\n2", + "v13:\n0:-1:j:\n1", "v14:\n0:+1:k:\n0", "v15:\n1:+1:j:\n1", "v16:\n1:-1:j:\n2", "v17:\n0:-1:k:\n1", + "v18:\n0:+1:k:\n0", "v19:\n0:+1:l:\n1", "v20:\n2:+1:j:\n2", "v20:\n3:+1:n:\n3", "v21:\n1:+1:k:\n4", + "v21:\n2:+1:m:\n5", "v22:\n0:-6:lkmkjn,\n0:+1:h:\n6", "v23:\n1:+1:e:\n1", "v23:\n2:+1:l:\n2", "v23:\n3:+1:l:\n3", + "v24:\n4:+1:o:\n4", "v24:\n5:+1:m:\n5", "v25:\n6:+1:y:\n6", "v25:\n7:+1:n:\n7", "v25:\n8:+1:a:\n8", + "v26:\n9:+1:m:\n9", "v26:\n10:+1:e:\n10", "v26:\n11:+1:i:\n11", "v27:\n0:+1:h:\n12", "v28:\n13:+1:s:\n13", + "v28:\n14:+1:b:\n14", "v28:\n15:+1:e:\n15", "v28:\n16:+1:n:\n16", "v29:\n1:+1:e:\n17", "v29:\n2:+1:y:\n18", + "v30:\n3:+1:a:\n19", "v31:\n0:-20:heyahellomynameisben:\n20", "v32:\n0:+1:t:\n0", "v33:\n1:+1:r:\n1", "v34:\n1:-1:r:\n2", + "v35:\n0:-1:t:\n1", "v36:\n0:+1:t:\n0", "v37:\n1:+1:e:\n1", "v38:\n2:+1:s:\n2", "v39:\n3:+1:t:\n3", + "v40:\n3:-1:t:\n4", "v41:\n2:-1:s:\n3", "v42:\n1:-1:e:\n2", "v43:\n0:-1:t:\n1", "v44:\n0:+1:a:\n0", + "v45:\n0:+1:%3B:\n1", "v46:\n2:+1:s:\n2", "v47:\n3:+1:d:\n3", "v47:\n4:+1:f:\n4", "v48:\n1:+1:l:\n5", + "v48:\n2:+1:k:\n6", "v48:\n3:+1:j:\n7", "v49:\n8:+1:a:\n8", "v50:\n9:+1:s:\n9", "v50:\n10:+1:d:\n10", + "v51:\n11:+1:f:\n11", "v52:\n0:-12:%3Blkjasdfasdf,\n0:+12:%3Blkjasdfasdf:\n12", "v53:\n0:-12:%3Blkjasdfasdf,\n0:+1:t:\n12", "v54:\n1:+1:e:\n1", "v55:\n2:+1:s:\n2", + "v56:\n3:+1:t:\n3", "v57:\n1:-1:e:\n4", "v58:\n0:-1:t:\n3", "v59:\n0:+1:t:\n2", "v60:\n1:+1:e:\n3", + "v61:\n1:-1:e:\n4", "v62:\n0:-1:t:\n3", "v63:\n0:+1:t:\n2", "v64:\n1:+1:e:\n3", "v65:\n3:-1:t:\n4", + "v66:\n2:-1:s:\n3", "v67:\n1:-1:e:\n2", "v68:\n0:-1:t:\n1", "v69:\n0:+1:t:\n0", "v70:\n1:+1:e:\n1", + "v71:\n2:+1:s:\n2", "v72:\n3:+1:t:\n3", "v73:\n3:-1:t:\n4", "v74:\n0:-1:t:\n3", "v75:\n1:-1:s:\n2", + "v76:\n0:-1:e:\n1", "v77:\n0:+1:a:\n0", "v78:\n0:+1:%3B:\n1", "v79:\n2:+1:d:\n2", "v79:\n3:+1:s:\n3", + "v80:\n4:+1:f:\n4", "v81:\n5:+1:a:\n5", "v82:\n6:+1:s:\n6", "v82:\n7:+1:d:\n7", "v82:\n8:+1:f:\n8", + "v83:\n1:+1:l:\n9", "v83:\n2:+1:k:\n10", "v83:\n3:+1:j:\n11", "v83:\n4:+1:%3B:\n12", "v83:\n5:+1:l:\n13", + "v83:\n6:+1:k:\n14", "v83:\n7:+1:j:\n15", "v84:\n8:+1:%0A:\n16", "v85:\n9:+1:%0A:\n17", "v86:\n10:-8:adsfasdf,\n10:+1:h:\n18", + "v87:\n11:+1:e:\n11", "v88:\n12:+1:l:\n12", "v89:\n13:+1:l:\n13", "v90:\n14:+1:o:\n14", "v91:\n15:+1:+:\n15", + "v92:\n16:+1:m:\n16", "v93:\n17:+1:y:\n17", "v94:\n18:+1:+:\n18", "v95:\n19:+1:n:\n19", "v96:\n20:+1:a:\n20", + "v97:\n21:+1:m:\n21", "v98:\n22:+1:e:\n22", "v99:\n23:+1:+:\n23", "v100:\n24:+1:i:\n24", "v101:\n25:+1:s:\n25", + "v102:\n26:+1:+:\n26", "v103:\n27:+1:b:\n27", "v104:\n28:+1:e:\n28", "v105:\n29:+1:n:\n29", "v106:\n30:+1:.:\n30", + "v107:\n27:-4:ben.:\n31", "v108:\n24:-3:is+:\n27", "v109:\n19:-5:name+:\n24", "v110:\n16:-3:my+:\n19", "v111:\n10:-6:hello+:\n16", + "v112:\n0:-8:%3Blkj%3Blkj,\n0:+1:H:\n10", "v113:\n1:+1:e:\n3", "v114:\n2:+1:l:\n4", "v115:\n3:+1:l:\n5", "v116:\n4:+1:o:\n6", + "v117:\n5:+1:+:\n7", "v118:\n6:+1:f:\n8", "v119:\n7:+1:r:\n9", "v120:\n8:+1:o:\n10", "v121:\n9:+1:m:\n11", + "v122:\n10:+1:+:\n12", "v123:\n11:+1:c:\n13", "v124:\n12:+1:o:\n14", "v125:\n13:+1:m:\n15", "v126:\n14:+1:p:\n16", + "v127:\n15:+1:2:\n17", "v128:\n16:+1:+:\n18", "v129:\n16:-1:+:\n19", "v130:\n18:+1:H:\n18", "v131:\n19:+1:e:\n19", + "v132:\n20:+1:l:\n20", "v133:\n21:+1:l:\n21", "v134:\n22:+1:o:\n22", "v135:\n23:+1:+:\n23", "v136:\n24:+1:f:\n24", + "v137:\n25:+1:r:\n25", "v138:\n26:+1:o:\n26", "v139:\n27:+1:m:\n27", "v140:\n28:+1:+:\n28", "v141:\n29:+1:c:\n29", + "v142:\n30:+1:o:\n30", "v143:\n31:+1:m:\n31", "v144:\n32:+1:p:\n32", "v145:\n33:+1:1:\n33", "v146:\n29:-5:comp1:\n34", + "v147:\n24:-5:from+:\n29", "v148:\n18:-6:Hello+:\n24", "v149:\n0:-16:Hello+from+comp2,\n0:+1:a:\n18", "v150:\n0:-1:a:\n3", "v151:\n0:+1:a:\n2", + "v152:\n1:+1:s:\n3", "v153:\n2:+1:d:\n4", "v154:\n5:+1:H:\n5", "v155:\n3:+1:f:\n6", "v156:\n4:+1:a:\n7", + "v157:\n5:+1:s:\n8", "v158:\n6:+1:d:\n9", "v159:\n7:+1:f:\n10", "v160:\n8:+1:a:\n11", "v161:\n9:+1:s:\n12", + "v162:\n10:+1:d:\n13", "v163:\n11:+1:f:\n14", "v164:\n12:+1:a:\n15", "v165:\n16:+1:i:\n16", "v165:\n17:+1:+:\n17", + "v165:\n18:+1:m:\n18", "v165:\n19:+1:y:\n19", "v166:\n13:+1:s:\n20", "v166:\n14:+1:d:\n21", "v167:\n15:+1:f:\n22", + "v168:\n16:+1:a:\n23", "v169:\n17:+1:s:\n24", "v170:\n18:+1:d:\n25", "v171:\n26:+1:+:\n26", "v171:\n27:+1:n:\n27", + "v171:\n28:+1:a:\n28", "v171:\n29:+1:m:\n29", "v172:\n19:+1:f:\n30", "v173:\n20:+1:a:\n31", "v174:\n32:+1:e:\n32", + "v174:\n33:+1:+:\n33", "v175:\n21:+1:s:\n34", "v176:\n22:+1:d:\n35", "v176:\n23:+1:f:\n36", "v177:\n37:+1:i:\n37", + "v178:\n24:+1:a:\n38", "v179:\n25:+1:s:\n39", "v180:\n26:+1:d:\n40", "v181:\n27:+1:f:\n41", "v182:\n28:+1:a:\n42", + "v183:\n43:+1:s:\n43", "v183:\n44:+1:+:\n44", "v183:\n45:+1:b:\n45", "v184:\n29:+1:s:\n46", "v184:\n30:+1:d:\n47", + "v184:\n31:+1:f:\n48", "v184:\n32:+1:a:\n49", "v185:\n33:+1:s:\n50", "v185:\n34:+1:d:\n51", "v186:\n35:+1:f:\n52", + "v187:\n36:+1:a:\n53", "v188:\n37:+1:s:\n54", "v189:\n38:+1:d:\n55", "v189:\n39:+1:f:\n56", "v190:\n56:-1:b:\n57", + "v191:\n40:+1:a:\n56", "v192:\n41:+1:s:\n57", "v193:\n58:+1:w:\n58", "v194:\n42:+1:d:\n59", "v194:\n43:+1:f:\n60", + "v194:\n44:+1:j:\n61", "v195:\n62:+1:l:\n62", "v196:\n45:+1:%3B:\n63", "v196:\n46:+1:l:\n64", "v197:\n47:+1:k:\n65", + "v197:\n48:+1:j:\n66", "v198:\n49:+1:a:\n67", "v199:\n50:+1:%3B:\n68", "v199:\n51:+1:o:\n69", "v200:\n52:+1:f:\n70", + "v200:\n53:+1:i:\n71", "v200:\n54:+1:w:\n72", "v200:\n55:+1:e:\n73", "v201:\n56:+1:o:\n74", "v202:\n57:+1:a:\n75", + "v203:\n58:+1:e:\n76", "v203:\n59:+1:j:\n77", "v204:\n60:+1:f:\n78", "v205:\n61:+1:o:\n79", "v206:\n62:+1:j:\n80", + "v207:\n63:+1:a:\n81", "v207:\n64:+1:d:\n82", "v207:\n65:+1:%3B:\n83", "v207:\n66:+1:s:\n84", "v208:\n67:+1:l:\n85", + "v208:\n68:+1:k:\n86", "v209:\n69:+1:j:\n87", "v210:\n70:+1:o:\n88", "v211:\n71:+1:w:\n89", "v211:\n72:+1:a:\n90", + "v212:\n73:+1:e:\n91", "v212:\n74:+1:i:\n92", "v212:\n75:+1:u:\n93", "v213:\n76:+1:p:\n94", "v213:\n77:+1:r:\n95", + "v214:\n77:-1:r:\n96", "v215:\n95:+1:k:\n95", "v215:\n96:+1:e:\n96", "v215:\n97:+1:j:\n97", "v215:\n98:+1:f:\n98", + "v215:\n99:+1:a:\n99", "v215:\n100:+1:s:\n100", "v215:\n93:-8:wlkejfas:\n101", "v215:\n90:-3:is+:\n93", "v216:\n76:-1:p:\n90", + "v217:\n75:-1:u:\n89", "v218:\n74:-1:i:\n88", "v219:\n73:-1:e:\n87", "v219:\n72:-1:a:\n86", "v220:\n71:-1:w:\n85", + "v220:\n70:-1:o:\n84", "v221:\n69:-1:j:\n83", "v221:\n68:-1:k:\n82", "v222:\n67:-1:l:\n81", "v222:\n66:-1:s:\n80", + "v223:\n65:-1:%3B:\n79", "v223:\n64:-1:d:\n78", "v224:\n63:-1:a:\n77", "v224:\n62:-1:j:\n76", "v225:\n61:-1:o:\n75", + "v225:\n60:-1:f:\n74", "v226:\n59:-1:j:\n73", "v226:\n58:-1:e:\n72", "v227:\n57:-1:a:\n71", "v227:\n56:-1:o:\n70", + "v228:\n55:-1:e:\n69", "v228:\n54:-1:w:\n68", "v229:\n53:-1:i:\n67", "v229:\n52:-1:f:\n66", "v230:\n51:-1:o:\n65", + "v230:\n50:-1:%3B:\n64", "v231:\n49:-1:a:\n63", "v231:\n48:-1:j:\n62", "v232:\n47:-1:k:\n61", "v232:\n46:-1:l:\n60", + "v233:\n45:-1:%3B:\n59", "v233:\n44:-1:j:\n58", "v234:\n43:-1:f:\n57", "v234:\n42:-1:d:\n56", "v235:\n41:-1:s:\n55", + "v235:\n40:-1:a:\n54", "v236:\n39:-1:f:\n53", "v236:\n38:-1:d:\n52", "v237:\n37:-1:s:\n51", "v237:\n36:-1:a:\n50", + "v238:\n35:-1:f:\n49", "v238:\n34:-1:d:\n48", "v239:\n33:-1:s:\n47", "v239:\n32:-1:a:\n46", "v239:\n31:-1:f:\n45", + "v239:\n30:-1:d:\n44", "v240:\n29:-1:s:\n43", "v240:\n28:-1:a:\n42", "v240:\n27:-1:f:\n41", "v241:\n26:-1:d:\n40", + "v241:\n25:-1:s:\n39", "v242:\n24:-1:a:\n38", "v242:\n23:-1:f:\n37", "v242:\n22:-1:d:\n36", "v243:\n21:-1:s:\n35", + "v243:\n20:-1:a:\n34", "v244:\n19:-1:f:\n33", "v244:\n18:-1:d:\n32", "v245:\n17:-1:s:\n31", "v245:\n16:-1:a:\n30", + "v246:\n15:-1:f:\n29", "v246:\n14:-1:d:\n28", "v247:\n13:-1:s:\n27", "v247:\n12:-1:a:\n26", "v248:\n11:-1:f:\n25", + "v248:\n10:-1:d:\n24", "v249:\n9:-1:s:\n23", "v249:\n8:-1:a:\n22", "v250:\n7:-1:f:\n21", "v250:\n6:-1:d:\n20", + "v251:\n5:-1:s:\n19", "v251:\n4:-1:a:\n18", "v252:\n3:-1:f:\n17", "v252:\n2:-1:d:\n16", "v253:\n1:-1:s:\n15", + "v254:\n0:-1:a:\n14", "v255:\n2:-11:Hi+my+name+:\n13", "v256:\n2:+1:H:\n2", "v257:\n3:+1:e:\n3", "v258:\n4:+1:l:\n4", + "v259:\n5:+1:l:\n5", "v260:\n6:+1:o:\n6", "v261:\n7:+1:+:\n7", "v262:\n8:+1:m:\n8", "v263:\n9:+1:y:\n9", + "v264:\n10:+1:+:\n10", "v265:\n11:+1:n:\n11", "v266:\n12:+1:a:\n12", "v267:\n13:+1:m:\n13", "v268:\n14:+1:+:\n14", + "v269:\n15:+1:e:\n15", "v270:\n0:+1:L:\n16", "v271:\n1:+1:p:\n17", "v272:\n17:-1:e:\n18", "v273:\n2:+1:o:\n17", + "v273:\n3:+1:r:\n18", "v274:\n18:-1:+:\n19", "v275:\n18:+1:e:\n18", "v276:\n19:+1:+:\n19", "v277:\n20:+1:i:\n20", + "v278:\n21:+1:s:\n21", "v279:\n22:+1:+:\n22", "v280:\n3:-1:r:\n23", "v281:\n22:+1:b:\n22", "v282:\n2:-1:o:\n23", + "v283:\n22:+1:e:\n22", "v284:\n1:-1:p:\n23", "v285:\n22:+1:n:\n22", "v286:\n23:+1:.:\n23", "v287:\n1:+1:r:\n24", + "v288:\n25:+1:+:\n25", "v289:\n2:+1:o:\n26", "v290:\n27:+1:C:\n27", "v291:\n2:-1:o:\n28", "v292:\n27:+1:a:\n27", + "v293:\n1:-1:r:\n28", "v294:\n27:+1:n:\n27", "v295:\n28:+1:+:\n28", "v296:\n29:+1:i:\n29", "v297:\n1:+1:o:\n30", + "v298:\n2:+1:r:\n31", "v299:\n3:+1:e:\n32", "v300:\n32:-1:i:\n33", "v301:\n4:+1:m:\n32", "v302:\n5:+1:+:\n33", + "v303:\n6:+1:i:\n34", "v304:\n7:+1:p:\n35", "v305:\n8:+1:s:\n36", "v306:\n9:+1:u:\n37", "v307:\n10:+1:m:\n38", + "v308:\n11:+1:+:\n39", "v309:\n12:+1:d:\n40", "v310:\n13:+1:o:\n41", "v311:\n14:+1:l:\n42", "v312:\n15:+1:o:\n43", + "v313:\n16:+1:r:\n44", "v313:\n17:+1:+:\n45", "v314:\n18:+1:s:\n46", "v314:\n19:+1:i:\n47", "v314:\n20:+1:t:\n48", + "v315:\n21:+1:+:\n49", "v316:\n22:+1:a:\n50", "v317:\n23:+1:m:\n51", "v318:\n24:+1:e:\n52", "v319:\n25:+1:t:\n53", + "v320:\n26:+1:+:\n54", "v321:\n55:+1:y:\n55", "v321:\n56:+1:o:\n56", "v321:\n57:+1:u:\n57", "v321:\n58:+1:+:\n58", + "v321:\n59:+1:s:\n59", "v321:\n60:+1:e:\n60", "v321:\n61:+1:e:\n61", "v321:\n62:+1:+:\n62", "v321:\n63:+1:t:\n63", + "v321:\n64:+1:h:\n64", "v321:\n65:+1:i:\n65", "v321:\n66:+1:s:\n66", "v321:\n67:+1:+:\n67", "v321:\n68:+1:c:\n68", + "v321:\n69:+1:o:\n69", "v321:\n70:+1:r:\n70", "v321:\n71:+1:r:\n71", "v321:\n72:+1:e:\n72", "v321:\n73:+1:c:\n73", + "v321:\n74:+1:t:\n74", "v321:\n75:+1:l:\n75", "v321:\n76:+1:y:\n76", "v321:\n77:+1:%3F:\n77", "v322:\n0:-78:Lorem+ipsum+dolor+sit+amet+%0A%0AHello+my+name+is+ben.+Can+you+see+this+correctly%3F:\n78", + "v323:\n0:+1:t:\n0", "v324:\n1:+1:e:\n1", "v325:\n2:+1:s:\n2", "v326:\n3:+1:t:\n3", "v327:\n4:+1:i:\n4", + "v328:\n5:+1:n:\n5", "v329:\n6:+1:g:\n6", "v330:\n6:-1:g:\n7", "v331:\n5:-1:n:\n6", "v332:\n4:-1:i:\n5", + "v333:\n3:-1:t:\n4", "v334:\n2:-1:s:\n3", "v335:\n1:-1:e:\n2", "v336:\n0:-1:t:\n1", "v337:\n0:+1:%0A:\n0", + "v338:\n1:+1:O:\n1", "v339:\n2:+1:k:\n2", "v340:\n3:+1:.:\n3", "v341:\n4:+1:+:\n4", "v342:\n5:+1:T:\n5", + "v343:\n6:+1:h:\n6", "v344:\n7:+1:i:\n7", "v345:\n8:+1:s:\n8", "v346:\n9:+1:+:\n9", "v347:\n10:+1:i:\n10", + "v348:\n11:+1:s:\n11", "v349:\n12:+1:+:\n12", "v350:\n13:+1:a:\n13", "v351:\n14:+1:+:\n14", "v352:\n15:+1:n:\n15", + "v353:\n0:+1:H:\n16", "v354:\n16:-1:n:\n17", "v355:\n1:+1:i:\n16", "v356:\n16:-1:+:\n17", "v357:\n2:+1:+:\n16", + "v358:\n3:+1:t:\n17", "v359:\n18:+1:n:\n18", "v360:\n4:+1:h:\n19", "v361:\n5:+1:i:\n20", "v362:\n6:+1:s:\n21", + "v363:\n7:+1:+:\n22", "v364:\n8:+1:i:\n23", "v365:\n9:+1:s:\n24", "v366:\n10:+1:+:\n25", "v367:\n11:+1:m:\n26", + "v368:\n12:+1:y:\n27", "v369:\n13:+1:+:\n28", "v370:\n14:+1:t:\n29", "v371:\n15:+1:y:\n30", "v372:\n16:+1:p:\n31", + "v373:\n17:+1:i:\n32", "v374:\n18:+1:n:\n33", "v375:\n19:+1:g:\n34", "v376:\n20:+1:+:\n35", "v377:\n21:+1:e:\n36", + "v378:\n22:+1:x:\n37", "v379:\n23:+1:a:\n38", "v380:\n24:+1:m:\n39", "v381:\n25:+1:p:\n40", "v382:\n26:+1:l:\n41", + "v383:\n27:+1:e:\n42", "v384:\n28:+1:s:\n43", "v385:\n44:+1:o:\n44", "v385:\n45:+1:t:\n45", "v385:\n46:+1:h:\n46", + "v385:\n47:+1:e:\n47", "v385:\n48:+1:r:\n48", "v385:\n49:+1:+:\n49", "v385:\n50:+1:t:\n50", "v385:\n51:+1:e:\n51", + "v385:\n52:+1:s:\n52", "v385:\n53:+1:t:\n53", "v385:\n54:+1:+:\n54", "v385:\n55:+1:o:\n55", "v385:\n56:+1:f:\n56", + "v385:\n57:+1:+:\n57", "v385:\n58:+1:t:\n58", "v385:\n59:+1:h:\n59", "v385:\n60:+1:e:\n60", "v385:\n61:+1:+:\n61", + "v385:\n62:+1:s:\n62", "v385:\n63:+1:y:\n63", "v385:\n64:+1:n:\n64", "v385:\n65:+1:c:\n65", "v385:\n66:+1:h:\n66", + "v385:\n67:+1:r:\n67", "v385:\n68:+1:o:\n68", "v385:\n69:+1:n:\n69", "v385:\n70:+1:i:\n70", "v385:\n71:+1:z:\n71", + "v385:\n72:+1:a:\n72", "v385:\n73:+1:t:\n73", "v385:\n74:+1:i:\n74", "v386:\n75:+1:o:\n75", "v387:\n76:+1:n:\n76", + "v388:\n77:+1:+:\n77", "v389:\n78:+1:e:\n78", "v390:\n79:+1:x:\n79", "v391:\n80:+1:a:\n80", "v392:\n81:+1:m:\n81", + "v393:\n82:+1:p:\n82", "v394:\n29:+1:j:\n83", "v395:\n84:+1:l:\n84", "v396:\n30:+1:o:\n85", "v397:\n86:+1:e:\n86", + "v398:\n87:+1:.:\n87", "v399:\n88:+1:+:\n88", "v400:\n30:-1:o:\n89", "v401:\n29:-1:j:\n88", "v402:\n87:+1:W:\n87", + "v403:\n29:+1:+:\n88", "v404:\n89:+1:h:\n89", "v405:\n90:+1:a:\n90", "v406:\n91:+1:t:\n91", "v407:\n92:+1:+:\n92", + "v408:\n30:+1:n:\n93", "v409:\n31:+1:l:\n94", "v410:\n90:-5:What+:\n95", "v411:\n32:+1:z:\n90", "v412:\n82:-9:example.+:\n91", + "v413:\n32:-1:z:\n82", "v414:\n31:-1:l:\n81", "v415:\n80:+1:.:\n80", "v416:\n30:-1:n:\n81", "v417:\n79:-1:.:\n80", + "v418:\n78:-1:+:\n79", "v419:\n77:-1:n:\n78", "v420:\n30:+1:b:\n77", "v421:\n31:+1:l:\n78", "v422:\n32:+1:a:\n79", + "v423:\n33:+1:h:\n80", "v424:\n34:+1:+:\n81", "v425:\n82:+1:n:\n82", "v426:\n35:+1:b:\n83", "v427:\n84:+1:.:\n84", + "v428:\n36:+1:l:\n85", "v429:\n37:+1:a:\n86", "v430:\n87:+1:+:\n87", "v431:\n38:+1:h:\n88", "v432:\n39:+1:+:\n89", + "v433:\n90:+1:I:\n90", "v434:\n40:+1:b:\n91", "v435:\n92:+1:t:\n92", "v436:\n41:+1:l:\n93", "v437:\n42:+1:a:\n94", + "v438:\n95:+1:+:\n95", "v438:\n96:+1:s:\n96", "v439:\n43:+1:h:\n97", "v440:\n44:+1:+:\n98", "v441:\n99:+1:e:\n99", + "v441:\n100:+1:e:\n100", "v441:\n101:+1:m:\n101", "v441:\n102:+1:s:\n102", "v442:\n103:+1:+:\n103", "v443:\n104:+1:l:\n104", + "v444:\n105:+1:i:\n105", "v445:\n45:+1:h:\n106", "v446:\n46:+1:e:\n107", "v447:\n108:+1:k:\n108", "v448:\n47:+1:y:\n109", + "v449:\n110:+1:e:\n110", "v450:\n111:+1:+:\n111", "v451:\n112:+1:n:\n112", "v452:\n113:+1:o:\n113", "v453:\n47:-1:y:\n114", + "v454:\n112:-1:o:\n113", "v455:\n47:+1:l:\n112", "v456:\n48:+1:l:\n113", "v457:\n113:-1:n:\n114", "v458:\n49:+1:o:\n113", + "v459:\n114:+1:f:\n114", "v460:\n50:+1:+:\n115", "v461:\n116:+1:a:\n116", "v462:\n117:+1:i:\n117", "v463:\n51:+1:w:\n118", + "v464:\n119:+1:r:\n119", "v465:\n52:+1:o:\n120", "v466:\n121:+1:+:\n121", "v467:\n53:+1:r:\n122", "v468:\n54:+1:l:\n123", + "v469:\n55:+1:d:\n124", "v470:\n125:+1:s:\n125", "v470:\n126:+1:y:\n126", "v470:\n127:+1:n:\n127", "v471:\n56:+1:%21:\n128", + "v472:\n129:+1:c:\n129", "v473:\n130:+1:+:\n130", "v474:\n131:+1:h:\n131", "v475:\n132:+1:e:\n132", "v476:\n133:+1:l:\n133", + "v477:\n134:+1:s:\n134", "v478:\n57:+1:+:\n135", "v479:\n136:+1:p:\n136", "v480:\n136:-1:p:\n137", "v481:\n135:-1:s:\n136", + "v482:\n135:+1:p:\n135", "v483:\n136:+1:s:\n136", "v484:\n137:+1:e:\n137", "v485:\n138:+1:+:\n138", "v486:\n139:+1:s:\n139", + "v487:\n140:+1:o:\n140", "v488:\n141:+1:m:\n141", "v489:\n142:+1:e:\n142", "v490:\n58:+1:j:\n143", "v491:\n144:+1:w:\n144", + "v492:\n145:+1:h:\n145", "v493:\n146:+1:a:\n146", "v494:\n147:+1:t:\n147", "v495:\n148:+1:.:\n148", "v496:\n149:+1:+:\n149", + "v497:\n140:-10:somewhat.+:\n150", "v498:\n133:-7:helpse+:\n140", "v499:\n128:-5:sync+:\n133", "v500:\n128:+1:h:\n128", "v501:\n129:+1:e:\n129", + "v502:\n130:+1:l:\n130", "v503:\n131:+1:p:\n131", "v504:\n132:+1:s:\n132", "v505:\n133:+1:+:\n133", "v506:\n134:+1:s:\n134", + "v507:\n135:+1:o:\n135", "v508:\n136:+1:m:\n136", "v509:\n137:+1:w:\n137", "v510:\n138:+1:e:\n138", "v511:\n138:-1:e:\n139", + "v512:\n137:-1:w:\n138", "v513:\n137:+1:e:\n137", "v514:\n138:+1:w:\n138", "v515:\n139:+1:h:\n139", "v516:\n140:+1:a:\n140", + "v517:\n59:+1:j:\n141", "v517:\n60:+1:s:\n142", "v517:\n61:+1:j:\n143", "v517:\n61:-1:j:\n144", "v517:\n60:-1:s:\n143", + "v517:\n59:-1:j:\n142", "v517:\n58:-1:j:\n141", "v517:\n58:+1:A:\n140", "v517:\n59:+1:r:\n141", "v517:\n60:+1:e:\n142", + "v517:\n61:+1:+:\n143", "v517:\n62:+1:w:\n144", "v517:\n63:+1:e:\n145", "v517:\n64:+1:+:\n146", "v517:\n65:+1:h:\n147", + "v517:\n66:+1:a:\n148", "v517:\n67:+1:v:\n149", "v518:\n150:+1:t:\n150", "v519:\n68:+1:e:\n151", "v520:\n68:-1:e:\n152", + "v521:\n68:+1:i:\n151", "v522:\n69:+1:n:\n152", "v523:\n70:+1:g:\n153", "v524:\n154:+1:.:\n154", "v525:\n71:+1:+:\n155", + "v526:\n72:+1:f:\n156", "v527:\n73:+1:u:\n157", "v528:\n74:+1:n:\n158", "v529:\n159:+1:+:\n159", "v530:\n75:+1:+:\n160", + "v531:\n76:+1:y:\n161", "v532:\n77:+1:e:\n162", "v533:\n162:-1:+:\n163", "v534:\n78:+1:t:\n162", "v535:\n163:+1:%0A:\n163", + "v536:\n164:+1:%0A:\n164", "v537:\n79:+1:%21:\n165", "v538:\n166:+1:I:\n166", "v539:\n167:+1:+:\n167", "v540:\n168:+1:t:\n168", + "v541:\n169:+1:h:\n169", "v542:\n79:-1:%21:\n170", "v543:\n169:+1:i:\n169", "v544:\n170:+1:n:\n170", "v545:\n171:+1:k:\n171", + "v546:\n172:+1:+:\n172", "v547:\n79:+1:%3F:\n173", "v548:\n174:+1:i:\n174", "v549:\n175:+1:t:\n175", "v550:\n176:+1:+:\n176", + "v551:\n177:+1:a:\n177", "v552:\n177:-1:a:\n178", "v553:\n176:-1:+:\n177", "v554:\n175:-1:t:\n176", "v555:\n174:-1:i:\n175", + "v556:\n174:+1:t:\n174", "v557:\n175:+1:h:\n175", "v558:\n176:+1:e:\n176", "v559:\n177:+1:r:\n177", "v560:\n178:+1:e:\n178", + "v561:\n179:+1:+:\n179", "v562:\n180:+1:i:\n180", "v563:\n181:+1:s:\n181", "v564:\n182:+1:+:\n182", "v565:\n183:+1:s:\n183", + "v566:\n184:+1:o:\n184", "v567:\n185:+1:m:\n185", "v568:\n186:+1:e:\n186", "v569:\n187:+1:+:\n187", "v570:\n188:+1:l:\n188", + "v571:\n189:+1:a:\n189", "v572:\n190:+1:g:\n190", "v573:\n191:+1:%2C:\n191", "v574:\n192:+1:+:\n192", "v575:\n193:+1:b:\n193", + "v576:\n194:+1:u:\n194", "v577:\n195:+1:t:\n195", "v578:\n196:+1:+:\n196", "v579:\n197:+1:i:\n197", "v580:\n198:+1:t:\n198", + "v581:\n199:+1:%27:\n199", "v582:\n200:+1:s:\n200", "v583:\n201:+1:+:\n201", "v584:\n202:+1:n:\n202", "v585:\n203:+1:o:\n203", + "v586:\n204:+1:t:\n204", "v587:\n205:+1:+:\n205", "v588:\n206:+1:t:\n206", "v589:\n207:+1:o:\n207", "v590:\n208:+1:o:\n208", + "v591:\n209:+1:+:\n209", "v592:\n210:+1:b:\n210", "v593:\n211:+1:a:\n211", "v594:\n212:+1:d:\n212", "v595:\n213:+1:.:\n213", + "v596:\n214:+1:+:\n214", "v597:\n215:+1:H:\n215", "v598:\n216:+1:o:\n216", "v599:\n217:+1:n:\n217", "v600:\n218:+1:e:\n218", + "v601:\n219:+1:s:\n219", "v602:\n220:+1:t:\n220", "v603:\n221:+1:l:\n221", "v604:\n222:+1:y:\n222", "v605:\n223:+1:+:\n223", + "v606:\n224:+1:i:\n224", "v607:\n225:+1:%27:\n225", "v608:\n226:+1:m:\n226", "v609:\n227:+1:+:\n227", "v610:\n228:+1:h:\n228", + "v611:\n229:+1:a:\n229", "v611:\n230:+1:p:\n230", "v612:\n231:+1:p:\n231", "v613:\n232:+1:y:\n232", "v614:\n233:+1:+:\n233", + "v614:\n234:+1:w:\n234", "v615:\n235:+1:i:\n235", "v616:\n80:+1:a:\n236", "v617:\n237:+1:t:\n237", "v618:\n81:+1:s:\n238", + "v619:\n239:+1:+:\n239", "v620:\n82:+1:d:\n240", "v621:\n241:+1:t:\n241", "v622:\n83:+1:f:\n242", "v623:\n243:+1:h:\n243", + "v623:\n244:+1:i:\n244", "v623:\n245:+1:s:\n245", "v624:\n83:-1:f:\n246", "v625:\n82:-1:d:\n245", "v626:\n244:+1:.:\n244", + "v626:\n245:+1:+:\n245", "v627:\n81:-1:s:\n246", "v628:\n80:-1:a:\n245", "v629:\n80:+1:+:\n244", "v630:\n81:+1:H:\n245", + "v631:\n82:+1:e:\n246", "v632:\n83:+1:l:\n247", "v633:\n84:+1:o:\n248", "v634:\n249:+1:%0A:\n249", "v635:\n84:-1:o:\n250", + "v636:\n249:+1:%0A:\n249", "v637:\n83:-1:l:\n250", "v638:\n82:-1:e:\n249", "v639:\n248:+1:C:\n248", "v640:\n249:+1:a:\n249", + "v641:\n81:-1:H:\n250", "v642:\n248:-1:a:\n249", "v643:\n247:-1:C:\n248", "v644:\n247:+1:H:\n247", "v645:\n248:+1:I:\n248", + "v646:\n249:+1:+:\n249", "v647:\n250:+1:C:\n250", "v648:\n251:+1:R:\n251", "v649:\n252:+1:H:\n252", "v650:\n253:+1:I:\n253", + "v651:\n253:-1:I:\n254", "v652:\n252:-1:H:\n253", "v653:\n251:-1:R:\n252", "v654:\n251:+1:H:\n251", "v655:\n252:+1:R:\n252", + "v656:\n253:+1:I:\n253", "v657:\n254:+1:S:\n254", "v658:\n255:+1:.:\n255", "v659:\n256:+1:%0A:\n256", "v660:\n257:+1:%0A:\n257", + "v661:\n258:+1:M:\n258", "v662:\n259:+1:a:\n259", "v663:\n260:+1:x:\n260", "v664:\n261:+1:+:\n261", "v665:\n262:+1:i:\n262", + "v666:\n263:+1:s:\n263", "v667:\n264:+1:+:\n264", "v668:\n265:+1:a:\n265", "v669:\n81:+1:a:\n266", "v670:\n267:+1:+:\n267", + "v671:\n268:+1:s:\n268", "v672:\n82:+1:s:\n269", "v673:\n270:+1:r:\n270", "v674:\n83:+1:d:\n271", "v675:\n271:-1:r:\n272", + "v676:\n84:+1:f:\n271", "v677:\n272:+1:c:\n272", "v678:\n84:-1:f:\n273", "v679:\n272:+1:r:\n272", "v680:\n273:+1:u:\n273", + "v681:\n83:-1:d:\n274", "v682:\n273:+1:.:\n273", "v683:\n82:-1:s:\n274", "v684:\n81:-1:a:\n273", "v685:\n271:-1:.:\n272", + "v686:\n271:+1:b:\n271", "v687:\n272:+1:.:\n272", "v688:\n81:+1:%0A:\n273", "v689:\n82:+1:%0A:\n274", "v690:\n83:+1:%0A:\n275", + "v691:\n84:+1:%0A:\n276", "v692:\n277:+1:%0A:\n277", "v693:\n85:+1:%0A:\n278", "v694:\n279:+1:%0A:\n279", "v695:\n86:+1:%0A:\n280", + "v696:\n281:+1:A:\n281", "v697:\n282:+1:+:\n282", "v698:\n86:-1:%0A:\n283", "v699:\n282:+1:h:\n282", "v700:\n85:-1:%0A:\n283", + "v701:\n282:+1:u:\n282", "v702:\n283:+1:g:\n283", "v703:\n84:-1:%0A:\n284", "v704:\n83:-1:%0A:\n283", "v705:\n83:+1:%0A:\n282", + "v706:\n83:-1:%0A:\n283", "v707:\n83:+1:%0A:\n282", "v708:\n83:-1:%0A:\n283", "v709:\n83:+1:%0A:\n282", "v710:\n83:-1:%0A:\n283", + "v711:\n83:+1:%0A:\n282", "v712:\n83:-1:%0A:\n283", "v713:\n83:+1:%0A:\n282", "v714:\n83:-1:%0A:\n283", "v715:\n83:+1:%0A:\n282", + "v716:\n83:-1:%0A:\n283", "v717:\n83:+1:%0A:\n282", "v718:\n83:-1:%0A:\n283", "v719:\n83:+1:%0A:\n282", "v720:\n83:-1:%0A:\n283", + "v721:\n282:+1:e:\n282", "v721:\n283:+1:+:\n283", "v721:\n284:+1:o:\n284", "v721:\n285:+1:n:\n285", "v721:\n286:+1:e:\n286", + "v721:\n287:+1:.:\n287", "v721:\n288:+1:%0A:\n288", "v722:\n289:+1:%0A:\n289", "v723:\n290:+1:T:\n290", "v724:\n291:+1:h:\n291", + "v725:\n292:+1:a:\n292", "v726:\n293:+1:t:\n293", "v726:\n294:+1:+:\n294", "v727:\n295:+1:r:\n295", "v728:\n296:+1:e:\n296", + "v729:\n297:+1:p:\n297", "v730:\n298:+1:l:\n298", "v731:\n299:+1:a:\n299", "v732:\n300:+1:y:\n300", "v733:\n301:+1:+:\n301", + "v734:\n302:+1:t:\n302", "v735:\n303:+1:h:\n303", "v736:\n304:+1:o:\n304", "v737:\n305:+1:.:\n305", "v738:\n83:+1:%09:\n306", + "v739:\n84:+1:%09:\n307", "v740:\n85:+1:%09:\n308", "v741:\n86:+1:%09:\n309", "v742:\n87:+1:%09:\n310", "v743:\n88:+1:%09:\n311", + "v744:\n89:+1:j:\n312", "v745:\n90:+1:a:\n313", "v746:\n91:+1:s:\n314", "v747:\n92:+1:d:\n315", "v747:\n93:+1:f:\n316", + "v748:\n94:+7:%0A%09%09%09%09%09%09:\n317", "v749:\n101:+1:o:\n324", "v750:\n102:+1:w:\n325", "v750:\n103:+1:e:\n326", "v750:\n104:+1:i:\n327", + "v751:\n105:+1:j:\n328", "v751:\n106:+1:r:\n329", "v752:\n107:+1:o:\n330", "v752:\n108:+1:w:\n331", "v752:\n109:+1:i:\n332", + "v752:\n110:+1:e:\n333", "v753:\n111:+1:u:\n334", "v753:\n112:+1:r:\n335", "v753:\n113:+1:o:\n336", "v753:\n114:+1:w:\n337", + "v754:\n115:+7:%0A%09%09%09%09%09%09:\n338", "v755:\n122:+1:w:\n345", "v755:\n123:+1:e:\n346", "v756:\n124:+1:r:\n347", "v756:\n125:+1:o:\n348", + "v757:\n126:+1:i:\n349", "v758:\n127:+1:w:\n350", "v758:\n128:+1:e:\n351", "v759:\n129:+1:o:\n352", "v759:\n130:+1:i:\n353", + "v760:\n131:+1:w:\n354", "v760:\n132:+1:e:\n355", "v760:\n133:+1:j:\n356", "v761:\n134:+1:r:\n357", "v762:\n135:+7:%0A%09%09%09%09%09%09:\n358", + "v763:\n142:+1:a:\n365", "v764:\n366:+1:S:\n366", "v764:\n367:+1:e:\n367", "v765:\n368:+1:e:\n368", "v766:\n368:-1:e:\n369", + "v767:\n142:-1:a:\n368", "v768:\n366:-1:e:\n367", "v769:\n365:-1:S:\n366", "v770:\n141:-1:%09:\n365", "v771:\n364:+1:%0A:\n364", + "v772:\n365:+1:%0A:\n365", "v773:\n366:+1:S:\n366", "v774:\n140:-1:%09:\n367", "v775:\n366:+1:o:\n366", "v776:\n367:+1:+:\n367", + "v777:\n139:-1:%09:\n368", "v778:\n367:+1:f:\n367", "v779:\n138:-1:%09:\n368", "v780:\n367:+1:a:\n367", "v781:\n368:+1:r:\n368", + "v782:\n369:+1:%2C:\n369", "v783:\n370:+1:+:\n370", "v784:\n137:-1:%09:\n371", "v785:\n370:+1:i:\n370", "v786:\n371:+1:t:\n371", + "v787:\n372:+1:+:\n372", "v788:\n373:+1:s:\n373", "v789:\n374:+1:e:\n374", "v790:\n136:-1:%09:\n375", "v791:\n374:+1:e:\n374", + "v792:\n375:+1:m:\n375", "v793:\n376:+1:s:\n376", "v794:\n377:+1:+:\n377", "v795:\n378:+1:g:\n378", "v796:\n379:+1:o:\n379", + "v797:\n380:+1:o:\n380", "v798:\n381:+1:d:\n381", "v799:\n382:+1:.:\n382", "v800:\n383:+1:%0A:\n383", "v801:\n384:+1:%0A:\n384", + "v802:\n385:+1:T:\n385", "v803:\n386:+1:h:\n386", "v804:\n387:+1:e:\n387", "v805:\n388:+1:r:\n388", "v806:\n389:+1:e:\n389", + "v807:\n390:+1:%27:\n390", "v808:\n391:+1:s:\n391", "v809:\n392:+1:+:\n392", "v810:\n393:+1:a:\n393", "v811:\n394:+1:+:\n394", + "v812:\n395:+1:l:\n395", "v813:\n396:+1:i:\n396", "v814:\n397:+1:t:\n397", "v815:\n398:+1:t:\n398", "v816:\n399:+1:l:\n399", + "v817:\n400:+1:e:\n400", "v818:\n401:+1:+:\n401", "v819:\n402:+1:l:\n402", "v820:\n403:+1:a:\n403", "v821:\n404:+1:g:\n404", + "v822:\n405:+1:%2C:\n405", "v823:\n406:+1:+:\n406", "v824:\n407:+1:b:\n407", "v825:\n408:+1:u:\n408", "v826:\n409:+1:t:\n409", + "v827:\n410:+1:+:\n410", "v828:\n411:+1:t:\n411", "v829:\n412:+1:h:\n412", "v829:\n413:+1:a:\n413", "v830:\n414:+1:t:\n414", + "v830:\n415:+1:%27:\n415", "v831:\n416:+1:s:\n416", "v832:\n417:+1:+:\n417", "v833:\n418:+1:f:\n418", "v834:\n419:+1:i:\n419", + "v835:\n420:+1:n:\n420", "v836:\n421:+1:e:\n421", "v837:\n422:+1:.:\n422", "v838:\n423:+1:%0A:\n423", "v839:\n424:+1:x:\n424", + "v840:\n425:+1:%60:\n425", "v841:\n424:-2:x%60:\n426", "v842:\n0:-424:Hi+this+is+my+typing+examples+blah+blah+blah+hello+world%21+Are+we+having+fun+yet%3F+%0A%0A%09%09%09%09%09%09jasdf%0A%09%09%09%09%09%09oweijrowieurow%0A%09%09%09%09%09%09weroiweoiwejr%0A%0AOk.+This+is+another+test+of+the+synchronization.+It+seems+like+fair+helps+somewhat.%0A%0AI+think+there+is+some+lag%2C+but+it%27s+not+too+bad.+Honestly+i%27m+happy+wit+this.+%0A%0AHI+CHRIS.%0A%0AMax+is+a+scrub.%0A%0AA+huge+one.%0A%0AThat+replay+tho.%0A%0ASo+far%2C+it+seems+good.%0A%0AThere%27s+a+little+lag%2C+but+that%27s+fine.%0A:\n424", "v843:\n0:+1:t:\n0", "v844:\n1:+1:e:\n1", + "v845:\n2:+1:s:\n2", "v846:\n3:+1:t:\n3", "v847:\n4:+1:i:\n4", "v848:\n5:+1:n:\n5", "v849:\n6:+1:g:\n6", + "v850:\n7:+1:+:\n7", "v851:\n8:+1:t:\n8", "v852:\n9:+1:e:\n9", "v853:\n10:+1:s:\n10", "v853:\n11:+1:t:\n11", + "v854:\n12:+1:i:\n12", "v854:\n13:+1:n:\n13", "v854:\n14:+1:g:\n14", "v855:\n15:+1:+:\n15", "v856:\n16:+1:t:\n16", + "v857:\n17:+1:e:\n17", "v858:\n18:+1:s:\n18", "v859:\n19:+1:t:\n19", "v860:\n20:+1:i:\n20", "v861:\n21:+1:n:\n21", + "v862:\n22:+1:g:\n22", "v863:\n23:+1:%0A:\n23", "v864:\n24:+1:%0A:\n24", "v865:\n25:+1:S:\n25", "v866:\n26:+1:e:\n26", + "v867:\n27:+1:e:\n27", "v868:\n28:+1:+:\n28", "v869:\n29:+1:i:\n29", "v870:\n30:+1:f:\n30", "v870:\n31:+1:+:\n31", + "v871:\n32:+1:t:\n32", "v871:\n33:+1:h:\n33", "v872:\n34:+1:i:\n34", "v873:\n35:+1:s:\n35", "v873:\n36:+1:+:\n36", + "v874:\n37:+1:w:\n37", "v875:\n38:+1:o:\n38", "v875:\n39:+1:r:\n39", "v876:\n40:+1:k:\n40", "v877:\n41:+1:s:\n41", + "v878:\n42:+1:.:\n42", "v879:\n43:+1:%0A:\n43", "v880:\n44:+1:%0A:\n44", "v881:\n45:+1:S:\n45", "v882:\n46:+1:o:\n46", + "v882:\n47:+1:+:\n47", "v883:\n48:+1:f:\n48", "v884:\n49:+1:a:\n49", "v885:\n50:+1:r:\n50", "v885:\n51:+1:+:\n51", + "v886:\n52:+1:s:\n52", "v886:\n53:+1:o:\n53", "v887:\n54:+1:+:\n54", "v888:\n55:+1:g:\n55", "v889:\n56:+1:o:\n56", + ), + }, + } + + for _, test := range tests { + patchedText, err := PatchText(test.baseText, test.patches) + require.Nil(t, err) + + consolidatedPatch, err := ConsolidatePatches(test.patches) + require.Nil(t, err) + + consolidatedPatchedText, err := PatchText(test.baseText, []*Patch{consolidatedPatch}) + require.Equal(t, patchedText, consolidatedPatchedText, "TestConsolidatePatchLong[%s]: Expected %s but got %s", test.desc, patchedText, consolidatedPatchedText) + } +} diff --git a/modules/patching/diff.go b/modules/patching/diff.go index 3dcaa76..cd214d0 100644 --- a/modules/patching/diff.go +++ b/modules/patching/diff.go @@ -26,6 +26,37 @@ func (slice Diffs) Swap(i, j int) { slice[i], slice[j] = slice[j], slice[i] } +// Simplify merges deletion slice within this patch. +// This does not merge insertions, because insertions within the same patch are not actually adjacent +// due to the character that is in between. +func (slice Diffs) Simplify() Diffs { + if len(slice) == 0 { + return slice + } + + result := Diffs{} + result = append(result, slice[0].clone()) + + for i, j := 1, 0; i < len(slice); i++ { + if slice[i] == nil { + break + } + curr := slice[i] + prev := result[j] + + if !curr.Insertion && !prev.Insertion && prev.StartIndex+prev.Length() == curr.StartIndex { + prev.Changes = prev.Changes + curr.Changes + } else if curr.Insertion && prev.Insertion && prev.StartIndex == curr.StartIndex { + prev.Changes = prev.Changes + curr.Changes + } else { + j++ + result = append(result, curr.clone()) + } + } + + return result +} + // Diff represents a single change in the document. type Diff struct { Insertion bool @@ -145,11 +176,6 @@ func (diff *Diff) ConvertToLF(base string) *Diff { return NewDiff(diff.Insertion, newStartIndex, newChanges) } -// Undo reverses this diff, producing a diff to undo the changes done by applying the diff. -func (diff *Diff) Undo() *Diff { - return NewDiff(!diff.Insertion, diff.StartIndex, diff.Changes) -} - // OffsetDiff shifts the start index of this diff by the provided offset func (diff *Diff) OffsetDiff(offset int) *Diff { return NewDiff(diff.Insertion, diff.StartIndex+offset, diff.Changes) @@ -167,131 +193,6 @@ func (diff *Diff) subChangesEndingAt(end int) *Diff { return NewDiff(diff.Insertion, diff.StartIndex, diff.Changes[:end]) } -func (diff *Diff) transform(others Diffs, othersHavePrecedence bool) Diffs { - intermediateDiffs := Diffs{} - intermediateDiffs = append(intermediateDiffs, diff) - - for _, other := range others { - newIntermediateDiffs := Diffs{} - for _, current := range intermediateDiffs { - switch { - // CASE 1: IndexA < IndexB - case other.StartIndex < current.StartIndex: - switch { - // CASES 1a, 1b: Ins - Ins, Ins - Rmv - case other.Insertion && current.Insertion, other.Insertion && !current.Insertion: - newIntermediateDiffs = doTransform(transformType2, newIntermediateDiffs, current, other) - // CASE 1c: Rmv - Ins - case !other.Insertion && current.Insertion: - newIntermediateDiffs = doTransform(transformType3, newIntermediateDiffs, current, other) - // CASE 1d: Rmv - Rmv - case !other.Insertion && !current.Insertion: - newIntermediateDiffs = doTransform(transformType4, newIntermediateDiffs, current, other) - // Fail; should never have gotten to here. - default: - panic(fmt.Sprintf("Got to invalid state while transforming [%s] on predessor [%+v], from list [%+v]", current.String(), other, others)) - } - // CASE 2: IndexA = IndexB - case other.StartIndex == current.StartIndex: - switch { - // CASE 2a: Ins - Ins - case other.Insertion && current.Insertion: - if othersHavePrecedence { - newIntermediateDiffs = doTransform(transformType2, newIntermediateDiffs, current, other) - } else { - newIntermediateDiffs = doTransform(transformType1, newIntermediateDiffs, current, other) - } - // CASE 2b: Ins - Rmv - case other.Insertion && !current.Insertion: - newIntermediateDiffs = doTransform(transformType2, newIntermediateDiffs, current, other) - // CASE 2c: Rmv - Ins - case !other.Insertion && current.Insertion: - newIntermediateDiffs = doTransform(transformType1, newIntermediateDiffs, current, other) - // CASE 2d: Rmv - Ins - case !other.Insertion && !current.Insertion: - newIntermediateDiffs = doTransform(transformType5, newIntermediateDiffs, current, other) - // Fail; should never have gotten to here. - default: - panic(fmt.Sprintf("Got to invalid state while transforming [%s] on predessor [%+v], from list [%+v]", current.String(), other, others)) - } - // CASE 3: IndexA > IndexB - case other.StartIndex > current.StartIndex: - switch { - // CASES 3a, 3c: Ins - Ins, Rmv - Ins - case other.Insertion && current.Insertion, !other.Insertion && current.Insertion: - newIntermediateDiffs = doTransform(transformType1, newIntermediateDiffs, current, other) - // CASE 3b: Ins - Rmv - case other.Insertion && !current.Insertion: - newIntermediateDiffs = doTransform(transformType6, newIntermediateDiffs, current, other) - // CASE 3d: Rmv - Rmv - case !other.Insertion && !current.Insertion: - newIntermediateDiffs = doTransform(transformType7, newIntermediateDiffs, current, other) - // Fail; should never have gotten to here. - default: - panic(fmt.Sprintf("Got to invalid state while transforming [%s] on predessor [%+v], from list [%+v]", current.String(), other, others)) - } - } - } - intermediateDiffs = newIntermediateDiffs - } - return intermediateDiffs -} - -func doTransform(transformFunc func(current, other *Diff) Diffs, currResults Diffs, current, other *Diff) Diffs { - return append(currResults, transformFunc(current, other)...) -} - -func transformType1(current, other *Diff) Diffs { - return Diffs{current} -} - -func transformType2(current, other *Diff) Diffs { - return Diffs{current.OffsetDiff(other.Length())} -} - -func transformType3(current, other *Diff) Diffs { - if (other.StartIndex + other.Length()) > current.StartIndex { - return Diffs{current.OffsetDiff(-(current.StartIndex - other.StartIndex))} - } - return Diffs{current.OffsetDiff(-other.Length())} -} - -func transformType4(current, other *Diff) Diffs { - if (other.StartIndex + other.Length()) <= current.StartIndex { - return Diffs{current.OffsetDiff(-other.Length())} - } else if (other.StartIndex + other.Length()) >= (current.StartIndex + current.Length()) { - return Diffs{} // Ignore change - } - overlap := other.StartIndex + other.Length() - current.StartIndex - newDiff := current.OffsetDiff(-other.Length() + overlap) - newDiff = newDiff.subChangesStartingFrom(overlap) - return Diffs{newDiff} -} - -func transformType5(current, other *Diff) Diffs { - if current.Length() > other.Length() { - return Diffs{current.subChangesStartingFrom(other.Length())} - } // Else do nothing; already done by previous patch. - return Diffs{} -} - -func transformType6(current, other *Diff) Diffs { - if (current.StartIndex + current.Length()) > other.StartIndex { - length1 := other.StartIndex - current.StartIndex - - diff1 := current.subChangesEndingAt(length1) - diff2 := current.subChangesStartingFrom(length1).OffsetDiff(other.Length()) - - return Diffs{diff1, diff2} - } - return Diffs{current} -} - -func transformType7(current, other *Diff) Diffs { - if (current.StartIndex + current.Length()) > other.StartIndex { - nonOverlap := other.StartIndex - current.StartIndex - - return Diffs{current.subChangesEndingAt(current.Length() - nonOverlap)} - } - return Diffs{current} +func (diff *Diff) clone() *Diff { + return NewDiff(diff.Insertion, diff.StartIndex, diff.Changes) } diff --git a/modules/patching/diff_test.go b/modules/patching/diff_test.go index 87da2d0..7abf3f0 100644 --- a/modules/patching/diff_test.go +++ b/modules/patching/diff_test.go @@ -6,6 +6,15 @@ import ( "github.com/stretchr/testify/require" ) +func getDiffOrDie(t *testing.T, diffStr string) *Diff { + diff, err := NewDiffFromString(diffStr) + if err != nil { + t.Fatalf("Failed to build diff from string %s", diffStr) + } + + return diff +} + func TestDiff_NewDiff(t *testing.T) { // Test addition diff := NewDiff(true, 1, "test") @@ -125,308 +134,3 @@ func TestDiff_ConvertBack(t *testing.T) { "}\r\n") require.Equal(t, diff.String(), CRLFDiff.String()) } - -func TestDiff_Undo(t *testing.T) { - diff, err := NewDiffFromString("0:+4:str1") - require.Nil(t, err) - newDiff := diff.Undo() - require.Equal(t, "0:-4:str1", newDiff.String()) - originalDiff := newDiff.Undo() - require.Equal(t, diff.String(), originalDiff.String()) - - diff, err = NewDiffFromString("1:-4:str2") - require.Nil(t, err) - newDiff = diff.Undo() - require.Equal(t, "1:+4:str2", newDiff.String()) - originalDiff = newDiff.Undo() - require.Equal(t, diff.String(), originalDiff.String()) -} - -func TestDiff_Transform1A(t *testing.T) { - diff1, err := NewDiffFromString("2:+4:str1") - require.Nil(t, err) - diff2, err := NewDiffFromString("4:+4:str2") - require.Nil(t, err) - result := diff2.transform(Diffs{diff1}, true) - require.Equal(t, 1, len(result)) - require.Equal(t, "8:+4:str2", result[0].String()) - - diff1, err = NewDiffFromString("0:+4:str1") - require.Nil(t, err) - diff2, err = NewDiffFromString("1:+4:str2") - require.Nil(t, err) - result = diff2.transform(Diffs{diff1}, true) - require.Equal(t, 1, len(result)) - require.Equal(t, "5:+4:str2", result[0].String()) -} - -func TestDiff_Transform1B(t *testing.T) { - diff1, err := NewDiffFromString("2:+4:str1") - require.Nil(t, err) - diff2, err := NewDiffFromString("4:-4:str2") - require.Nil(t, err) - result := diff2.transform(Diffs{diff1}, true) - require.Equal(t, 1, len(result)) - require.Equal(t, "8:-4:str2", result[0].String()) - - diff1, err = NewDiffFromString("0:+4:str1") - require.Nil(t, err) - diff2, err = NewDiffFromString("1:-4:str2") - require.Nil(t, err) - result = diff2.transform(Diffs{diff1}, true) - require.Equal(t, 1, len(result)) - require.Equal(t, "5:-4:str2", result[0].String()) -} - -func TestDiff_Transform1C(t *testing.T) { - // Test case 1: if (IndexA + LenA) > IndexB, shift B down by amoun of A that comes before IndexB - diff1, err := NewDiffFromString("2:-4:str1") - require.Nil(t, err) - diff2, err := NewDiffFromString("4:+4:str2") - require.Nil(t, err) - result := diff2.transform(Diffs{diff1}, true) - require.Equal(t, 1, len(result)) - require.Equal(t, "2:+4:str2", result[0].String()) - - diff1, err = NewDiffFromString("2:-10:longerstr1") - require.Nil(t, err) - diff2, err = NewDiffFromString("4:+4:str2") - require.Nil(t, err) - result = diff2.transform(Diffs{diff1}, true) - require.Equal(t, 1, len(result)) - require.Equal(t, "2:+4:str2", result[0].String()) - - // Test else case - diff1, err = NewDiffFromString("2:-4:str1") - require.Nil(t, err) - diff2, err = NewDiffFromString("6:+4:str2") - require.Nil(t, err) - result = diff2.transform(Diffs{diff1}, true) - require.Equal(t, 1, len(result)) - require.Equal(t, "2:+4:str2", result[0].String()) - - diff1, err = NewDiffFromString("2:-4:str1") - require.Nil(t, err) - diff2, err = NewDiffFromString("8:+4:str2") - require.Nil(t, err) - result = diff2.transform(Diffs{diff1}, true) - require.Equal(t, 1, len(result)) - require.Equal(t, "4:+4:str2", result[0].String()) -} - -func TestDiff_Transform1D(t *testing.T) { - // Test case 1: if IndexA + LenA < IndexB (No overlap), shift B down by LenA - diff1, err := NewDiffFromString("2:-4:str1") - require.Nil(t, err) - diff2, err := NewDiffFromString("8:-4:str2") - require.Nil(t, err) - result := diff2.transform(Diffs{diff1}, true) - require.Equal(t, 1, len(result)) - require.Equal(t, "4:-4:str2", result[0].String()) - - diff1, err = NewDiffFromString("2:-4:str1") - require.Nil(t, err) - diff2, err = NewDiffFromString("6:-4:str2") - require.Nil(t, err) - result = diff2.transform(Diffs{diff1}, true) - require.Equal(t, 1, len(result)) - require.Equal(t, "2:-4:str2", result[0].String()) - - // Test case 2: if IndexA + LenA >= IndexB + LenB, ignore B - diff1, err = NewDiffFromString("2:-10:longerstr1") - require.Nil(t, err) - diff2, err = NewDiffFromString("4:-4:str2") - require.Nil(t, err) - result = diff2.transform(Diffs{diff1}, true) - require.Equal(t, 0, len(result)) - - // Test else cases: if overlapping, shorten B by overlap, shift down by LenA - overlap - diff1, err = NewDiffFromString("2:-4:str1") - require.Nil(t, err) - diff2, err = NewDiffFromString("4:-4:str2") - require.Nil(t, err) - result = diff2.transform(Diffs{diff1}, true) - require.Equal(t, 1, len(result)) - require.Equal(t, "2:-2:r2", result[0].String()) -} - -func TestDiff_Transform2A(t *testing.T) { - diff1, err := NewDiffFromString("4:+4:str1") - require.Nil(t, err) - diff2, err := NewDiffFromString("4:+4:str2") - require.Nil(t, err) - result := diff2.transform(Diffs{diff1}, true) - require.Equal(t, 1, len(result)) - require.Equal(t, "8:+4:str2", result[0].String()) - - diff1, err = NewDiffFromString("0:+15:longTestString1") - require.Nil(t, err) - diff2, err = NewDiffFromString("0:+4:str2") - require.Nil(t, err) - result = diff2.transform(Diffs{diff1}, true) - require.Equal(t, 1, len(result)) - require.Equal(t, "15:+4:str2", result[0].String()) - - diff1, err = NewDiffFromString("4:+4:str1") - require.Nil(t, err) - diff2, err = NewDiffFromString("4:+4:str2") - require.Nil(t, err) - result = diff2.transform(Diffs{diff1}, false) - require.Equal(t, 1, len(result)) - require.Equal(t, "4:+4:str2", result[0].String()) - - diff1, err = NewDiffFromString("0:+15:longTestString1") - require.Nil(t, err) - diff2, err = NewDiffFromString("0:+4:str2") - require.Nil(t, err) - result = diff2.transform(Diffs{diff1}, false) - require.Equal(t, 1, len(result)) - require.Equal(t, "0:+4:str2", result[0].String()) -} - -func TestDiff_Transform2B(t *testing.T) { - diff1, err := NewDiffFromString("4:+4:str1") - require.Nil(t, err) - diff2, err := NewDiffFromString("4:-4:str2") - require.Nil(t, err) - result := diff2.transform(Diffs{diff1}, true) - require.Equal(t, 1, len(result)) - require.Equal(t, "8:-4:str2", result[0].String()) - - diff1, err = NewDiffFromString("0:+15:longTestString1") - require.Nil(t, err) - diff2, err = NewDiffFromString("0:-4:str2") - require.Nil(t, err) - result = diff2.transform(Diffs{diff1}, true) - require.Equal(t, 1, len(result)) - require.Equal(t, "15:-4:str2", result[0].String()) -} - -func TestDiff_Transform2C(t *testing.T) { - diff1, err := NewDiffFromString("4:-4:str1") - require.Nil(t, err) - diff2, err := NewDiffFromString("4:+4:str2") - require.Nil(t, err) - result := diff2.transform(Diffs{diff1}, true) - require.Equal(t, 1, len(result)) - require.Equal(t, "4:+4:str2", result[0].String()) - - diff1, err = NewDiffFromString("0:-15:longTestString1") - require.Nil(t, err) - diff2, err = NewDiffFromString("0:+4:str2") - require.Nil(t, err) - result = diff2.transform(Diffs{diff1}, true) - require.Equal(t, 1, len(result)) - require.Equal(t, "0:+4:str2", result[0].String()) -} - -func TestDiff_Transform2D(t *testing.T) { - // Test case 1: If LenB > LenA, remove LenA characters from B - diff1, err := NewDiffFromString("4:-4:str1") - require.Nil(t, err) - diff2, err := NewDiffFromString("4:-15:longTestString2") - require.Nil(t, err) - result := diff2.transform(Diffs{diff1}, true) - require.Equal(t, 1, len(result)) - require.Equal(t, "4:-11:TestString2", result[0].String()) - - // Test else case - if LenB <= LenA - diff1, err = NewDiffFromString("0:-15:longTestString1") - require.Nil(t, err) - diff2, err = NewDiffFromString("0:-4:str2") - require.Nil(t, err) - result = diff2.transform(Diffs{diff1}, true) - require.Equal(t, 0, len(result)) - - diff1, err = NewDiffFromString("4:-4:str1") - require.Nil(t, err) - diff2, err = NewDiffFromString("4:-4:str2") - require.Nil(t, err) - result = diff2.transform(Diffs{diff1}, true) - require.Equal(t, 0, len(result)) -} - -func TestDiff_Transform3A(t *testing.T) { - diff1, err := NewDiffFromString("5:+4:str1") - require.Nil(t, err) - diff2, err := NewDiffFromString("4:+4:str2") - require.Nil(t, err) - result := diff2.transform(Diffs{diff1}, true) - require.Equal(t, 1, len(result)) - require.Equal(t, "4:+4:str2", result[0].String()) - - diff1, err = NewDiffFromString("4:+4:str1") - require.Nil(t, err) - diff2, err = NewDiffFromString("0:+15:longTestString2") - require.Nil(t, err) - result = diff2.transform(Diffs{diff1}, true) - require.Equal(t, 1, len(result)) - require.Equal(t, "0:+15:longTestString2", result[0].String()) -} - -func TestDiff_Transform3B(t *testing.T) { - // Test case 1: If IndexB + LenB > IndexA, split B into two diffs - diff1, err := NewDiffFromString("5:+4:str1") - require.Nil(t, err) - diff2, err := NewDiffFromString("4:-8:longStr2") - require.Nil(t, err) - result := diff2.transform(Diffs{diff1}, true) - require.Equal(t, 2, len(result)) - require.Equal(t, "4:-1:l", result[0].String()) - require.Equal(t, "8:-7:ongStr2", result[1].String()) - - // Test else case: no change - diff1, err = NewDiffFromString("8:+4:str1") - require.Nil(t, err) - diff2, err = NewDiffFromString("0:-4:str2") - require.Nil(t, err) - result = diff2.transform(Diffs{diff1}, true) - require.Equal(t, 1, len(result)) - require.Equal(t, "0:-4:str2", result[0].String()) -} - -func TestDiff_Transform3C(t *testing.T) { - diff1, err := NewDiffFromString("9:-4:str1") - require.Nil(t, err) - diff2, err := NewDiffFromString("4:+4:str2") - require.Nil(t, err) - result := diff2.transform(Diffs{diff1}, true) - require.Equal(t, 1, len(result)) - require.Equal(t, "4:+4:str2", result[0].String()) - - diff1, err = NewDiffFromString("5:-15:longTestString1") - require.Nil(t, err) - diff2, err = NewDiffFromString("4:+4:str2") - require.Nil(t, err) - result = diff2.transform(Diffs{diff1}, true) - require.Equal(t, 1, len(result)) - require.Equal(t, "4:+4:str2", result[0].String()) -} - -func TestDiff_Transform3D(t *testing.T) { - // Test case 1: If IndexB + LenB > IndexA, shorten B by overlap (from end) - diff1, err := NewDiffFromString("6:-4:str1") - require.Nil(t, err) - diff2, err := NewDiffFromString("4:-4:str2") - require.Nil(t, err) - result := diff2.transform(Diffs{diff1}, true) - require.Equal(t, 1, len(result)) - require.Equal(t, "4:-2:st", result[0].String()) - - // Test else case: No change if no overlap - diff1, err = NewDiffFromString("8:-4:str1") - require.Nil(t, err) - diff2, err = NewDiffFromString("4:-4:str2") - require.Nil(t, err) - result = diff2.transform(Diffs{diff1}, true) - require.Equal(t, 1, len(result)) - require.Equal(t, "4:-4:str2", result[0].String()) - - diff1, err = NewDiffFromString("10:-4:str1") - require.Nil(t, err) - diff2, err = NewDiffFromString("4:-4:str2") - require.Nil(t, err) - result = diff2.transform(Diffs{diff1}, true) - require.Equal(t, 1, len(result)) - require.Equal(t, "4:-4:str2", result[0].String()) -} diff --git a/modules/patching/patch.go b/modules/patching/patch.go index 7e996f8..8b4a940 100644 --- a/modules/patching/patch.go +++ b/modules/patching/patch.go @@ -6,6 +6,7 @@ import ( "fmt" "strconv" "strings" + "unicode/utf8" ) // Patch represents a set of changes to a versioned document @@ -16,14 +17,33 @@ type Patch struct { // Changes is the list of changes that were applied to the document. // When patching, changes MUST be applied in order. Changes Diffs + + // DocLength is the length of the document prior to the application of this patch + DocLength int +} + +// GetPatches creates an array of patches, given the array of strings +func GetPatches(patchStrs []string) ([]*Patch, error) { + patches := make([]*Patch, len(patchStrs)) + for index, patchStr := range patchStrs { + patch, err := NewPatchFromString(patchStr) + if err != nil { + return nil, err + } + patches[index] = patch + } + return patches, nil } // NewPatch creates a new patch with the given parameters -func NewPatch(baseVersion int64, changes Diffs) *Patch { - return &Patch{ +func NewPatch(baseVersion int64, changes Diffs, docLength int) *Patch { + patch := &Patch{ BaseVersion: baseVersion, Changes: changes, + DocLength: docLength, } + + return patch.simplify() } // NewPatchFromString parses a patch from its given string representation @@ -32,7 +52,7 @@ func NewPatchFromString(str string) (*Patch, error) { patch := Patch{} parts := strings.Split(str, ":\n") - if len(parts) < 2 { + if len(parts) < 3 { return nil, errors.New("Invalid patch format") } @@ -41,14 +61,23 @@ func NewPatchFromString(str string) (*Patch, error) { } patch.BaseVersion, err = strconv.ParseInt(string(parts[0][1:]), 10, 64) + if err != nil { + return nil, err + } + docLen64, err := strconv.ParseInt(string(parts[2]), 10, 0) if err != nil { return nil, err } + patch.DocLength = int(docLen64) diffStrs := strings.Split(parts[1], ",\n") for _, diffStr := range diffStrs { + if len(diffStr) == 0 { + continue + } + newDiff, err := NewDiffFromString(diffStr) if err != nil { return nil, err @@ -56,7 +85,7 @@ func NewPatchFromString(str string) (*Patch, error) { patch.Changes = append(patch.Changes, newDiff) } - return &patch, nil + return patch.simplify(), nil } // ConvertToCRLF converts this patch from using LF to CRLF line separators given the base text to patch. @@ -67,7 +96,7 @@ func (patch *Patch) ConvertToCRLF(base string) *Patch { newChanges = append(newChanges, diff.ConvertToCRLF(base)) } - return NewPatch(patch.BaseVersion, newChanges) + return NewPatch(patch.BaseVersion, newChanges, utf8.RuneCountInString(strings.Replace(base, "\n", "\r\n", -1))) } // ConvertToLF converts this patch from using CRLF to LF line separators given the base text to patch. @@ -78,58 +107,7 @@ func (patch *Patch) ConvertToLF(base string) *Patch { newChanges = append(newChanges, diff.ConvertToLF(base)) } - return NewPatch(patch.BaseVersion, newChanges) -} - -// Undo reverses this patch, producing a patch to undo the changes done by applying the patch. -func (patch *Patch) Undo() *Patch { - newChanges := Diffs{} - - // This needs to be in reverse order, since all the diffs in a package will have been applied in order. - // The last diff will have been computed relative to the previous few. - for i := len(patch.Changes) - 1; i >= 0; i-- { - newChanges = append(newChanges, patch.Changes[i].Undo()) - } - - return NewPatch(patch.BaseVersion, newChanges) -} - -// TransformFromString does an Operational Transform against the other patches, creating a set -// of changes relative to previously applied changes. -func (patch *Patch) TransformFromString(others []string, othersHavePrecedence bool) (*Patch, error) { - patches := make([]*Patch, len(others)) - - for i, v := range others { - patch, err := NewPatchFromString(v) - if err != nil { - return nil, err - } - patches[i] = patch - } - - return patch.Transform(patches, othersHavePrecedence), nil -} - -// Transform does an Operational Transform against the other patches, creating a set -// of changes relative to previously applied changes. -func (patch *Patch) Transform(others []*Patch, othersHavePrecedence bool) *Patch { - intermediateDiffs := patch.Changes - maxVersionSeen := patch.BaseVersion - 1 - - for _, otherPatch := range others { - newIntermediateDiffs := Diffs{} - - for _, diff := range intermediateDiffs { - newIntermediateDiffs = append(newIntermediateDiffs, diff.transform(otherPatch.Changes, othersHavePrecedence)...) - } - - intermediateDiffs = newIntermediateDiffs - if maxVersionSeen < otherPatch.BaseVersion { - maxVersionSeen = otherPatch.BaseVersion - } - } - - return NewPatch(maxVersionSeen+1, intermediateDiffs) + return NewPatch(patch.BaseVersion, newChanges, utf8.RuneCountInString(strings.Replace(base, "\r\n", "\n", -1))) } func (patch *Patch) String() string { @@ -145,6 +123,13 @@ func (patch *Patch) String() string { buffer.WriteString(diff.String()) } } + buffer.WriteString(fmt.Sprintf(":\n%d", patch.DocLength)) return buffer.String() } + +func (patch *Patch) simplify() *Patch { + patch.Changes = patch.Changes.Simplify() + + return patch +} diff --git a/modules/patching/patch_test.go b/modules/patching/patch_test.go index 388a714..f28246d 100644 --- a/modules/patching/patch_test.go +++ b/modules/patching/patch_test.go @@ -3,40 +3,44 @@ package patching import ( "testing" + "github.com/kr/pretty" "github.com/stretchr/testify/require" ) func TestPatch_NewPatch(t *testing.T) { - patchString := "v1:\n3:-8:deletion,\n2:+6:insert" + patchString := "v1:\n3:-8:deletion,\n2:+6:insert:\n11" diff1 := NewDiff(false, 3, "deletion") diff2 := NewDiff(true, 2, "insert") - patch := NewPatch(1, Diffs{diff1, diff2}) + patch := NewPatch(1, Diffs{diff1, diff2}, 11) require.Equal(t, patchString, patch.String()) } func TestPatch_NewPatchFromString(t *testing.T) { - patch, err := NewPatchFromString("v6:\n3:-8:deletion,\n2:+6:insert") + patch, err := NewPatchFromString("v6:\n3:-8:deletion,\n2:+6:insert:\n11") require.Nil(t, err) require.Equal(t, int64(6), patch.BaseVersion) require.Equal(t, 2, len(patch.Changes)) require.Equal(t, "3:-8:deletion", patch.Changes[0].String()) require.Equal(t, "2:+6:insert", patch.Changes[1].String()) + require.Equal(t, 11, patch.DocLength) // Test insertion from string - patch, err = NewPatchFromString("v4:\n2:+1:a") + patch, err = NewPatchFromString("v4:\n2:+1:a:\n12") require.Nil(t, err) require.Equal(t, int64(4), patch.BaseVersion) require.Equal(t, 1, len(patch.Changes)) require.Equal(t, "2:+1:a", patch.Changes[0].String()) + require.Equal(t, 12, patch.DocLength) // Test insertion from string - patch, err = NewPatchFromString("v3:\n26:+2:ab,\n81:-3:cde") + patch, err = NewPatchFromString("v3:\n26:+2:ab,\n81:-3:cde:\n13") require.Nil(t, err) require.Equal(t, int64(3), patch.BaseVersion) require.Equal(t, 2, len(patch.Changes)) require.Equal(t, "26:+2:ab", patch.Changes[0].String()) require.Equal(t, "81:-3:cde", patch.Changes[1].String()) + require.Equal(t, 13, patch.DocLength) } func TestPatch_NewPatchFromStringInvalidFormats(t *testing.T) { @@ -67,137 +71,195 @@ func TestPatch_NewPatchFromStringInvalidFormats(t *testing.T) { } func TestPatch_ConvertToCRLF(t *testing.T) { - patch, err := NewPatchFromString("v0:\n0:+5:test%0A") + patch, err := NewPatchFromString("v0:\n0:+5:test%0A:\n12") require.Nil(t, err) newPatch := patch.ConvertToCRLF("\r\ntest") require.Equal(t, 1, len(newPatch.Changes)) - require.Equal(t, "v0:\n0:+6:test%0D%0A", newPatch.String()) + require.Equal(t, "v0:\n0:+6:test%0D%0A:\n7", newPatch.String()) - patch, err = NewPatchFromString("v0:\n1:+5:test%0A") + patch, err = NewPatchFromString("v0:\n1:+5:test%0A:\n12") require.Nil(t, err) newPatch = patch.ConvertToCRLF("\r\ntest") require.Equal(t, 1, len(newPatch.Changes)) - require.Equal(t, "v0:\n2:+6:test%0D%0A", newPatch.String()) + require.Equal(t, "v0:\n2:+6:test%0D%0A:\n7", newPatch.String()) - patch, err = NewPatchFromString("v0:\n2:+5:test%0A") + patch, err = NewPatchFromString("v0:\n2:+5:test%0A:\n12") require.Nil(t, err) newPatch = patch.ConvertToCRLF("\r\ntest") require.Equal(t, 1, len(newPatch.Changes)) - require.Equal(t, "v0:\n3:+6:test%0D%0A", newPatch.String()) + require.Equal(t, "v0:\n3:+6:test%0D%0A:\n7", newPatch.String()) - patch, err = NewPatchFromString("v0:\n7:+5:test%0A") + patch, err = NewPatchFromString("v0:\n7:+5:test%0A:\n12") require.Nil(t, err) newPatch = patch.ConvertToCRLF("\r\ntes\r\nt") require.Equal(t, 1, len(newPatch.Changes)) - require.Equal(t, "v0:\n9:+6:test%0D%0A", newPatch.String()) + require.Equal(t, "v0:\n9:+6:test%0D%0A:\n10", newPatch.String()) - patch, err = NewPatchFromString("v0:\n2:+5:test%0A,\n7:+5:test%0A") + patch, err = NewPatchFromString("v0:\n2:+5:test%0A,\n7:+5:test%0A:\n12") require.Nil(t, err) newPatch = patch.ConvertToCRLF("\r\ntes\r\nt") require.Equal(t, 2, len(newPatch.Changes)) - require.Equal(t, "v0:\n3:+6:test%0D%0A,\n9:+6:test%0D%0A", newPatch.String()) + require.Equal(t, "v0:\n3:+6:test%0D%0A,\n9:+6:test%0D%0A:\n10", newPatch.String()) - patch, err = NewPatchFromString("v0:\n2:+5:test%0A,\n7:+5:test%0A,\n0:+5:test%0A") + patch, err = NewPatchFromString("v0:\n2:+5:test%0A,\n7:+5:test%0A,\n0:+5:test%0A:\n12") require.Nil(t, err) newPatch = patch.ConvertToCRLF("\r\ntes\r\nt") require.Equal(t, 3, len(newPatch.Changes)) - require.Equal(t, "v0:\n3:+6:test%0D%0A,\n9:+6:test%0D%0A,\n0:+6:test%0D%0A", newPatch.String()) + require.Equal(t, "v0:\n3:+6:test%0D%0A,\n9:+6:test%0D%0A,\n0:+6:test%0D%0A:\n10", newPatch.String()) } func TestPatch_ConvertToLF(t *testing.T) { - patch, err := NewPatchFromString("v0:\n0:+6:test%0D%0A") + patch, err := NewPatchFromString("v0:\n0:+6:test%0D%0A:\n6") require.Nil(t, err) newPatch := patch.ConvertToLF("\r\ntest") require.Equal(t, 1, len(newPatch.Changes)) - require.Equal(t, "v0:\n0:+5:test%0A", newPatch.String()) + require.Equal(t, "v0:\n0:+5:test%0A:\n5", newPatch.String()) - patch, err = NewPatchFromString("v0:\n2:+6:test%0D%0A") + patch, err = NewPatchFromString("v0:\n2:+6:test%0D%0A:\n6") require.Nil(t, err) newPatch = patch.ConvertToLF("\r\ntest") require.Equal(t, 1, len(newPatch.Changes)) - require.Equal(t, "v0:\n1:+5:test%0A", newPatch.String()) + require.Equal(t, "v0:\n1:+5:test%0A:\n5", newPatch.String()) - patch, err = NewPatchFromString("v0:\n3:+6:test%0D%0A") + patch, err = NewPatchFromString("v0:\n3:+6:test%0D%0A:\n6") require.Nil(t, err) newPatch = patch.ConvertToLF("\r\ntest") require.Equal(t, 1, len(newPatch.Changes)) - require.Equal(t, "v0:\n2:+5:test%0A", newPatch.String()) + require.Equal(t, "v0:\n2:+5:test%0A:\n5", newPatch.String()) - patch, err = NewPatchFromString("v0:\n9:+6:test%0D%0A") + patch, err = NewPatchFromString("v0:\n9:+6:test%0D%0A:\n8") require.Nil(t, err) newPatch = patch.ConvertToLF("\r\ntes\r\nt") require.Equal(t, 1, len(newPatch.Changes)) - require.Equal(t, "v0:\n7:+5:test%0A", newPatch.String()) + require.Equal(t, "v0:\n7:+5:test%0A:\n6", newPatch.String()) - patch, err = NewPatchFromString("v0:\n3:+6:test%0D%0A,\n9:+6:test%0D%0A") + patch, err = NewPatchFromString("v0:\n3:+6:test%0D%0A,\n9:+6:test%0D%0A:\n8") require.Nil(t, err) newPatch = patch.ConvertToLF("\r\ntes\r\nt") require.Equal(t, 2, len(newPatch.Changes)) - require.Equal(t, "v0:\n2:+5:test%0A,\n7:+5:test%0A", newPatch.String()) + require.Equal(t, "v0:\n2:+5:test%0A,\n7:+5:test%0A:\n6", newPatch.String()) - patch, err = NewPatchFromString("v0:\n3:+6:test%0D%0A,\n9:+6:test%0D%0A,\n0:+6:test%0D%0A") + patch, err = NewPatchFromString("v0:\n3:+6:test%0D%0A,\n9:+6:test%0D%0A,\n0:+6:test%0D%0A:\n8") require.Nil(t, err) newPatch = patch.ConvertToLF("\r\ntes\r\nt") require.Equal(t, 3, len(newPatch.Changes)) - require.Equal(t, "v0:\n2:+5:test%0A,\n7:+5:test%0A,\n0:+5:test%0A", newPatch.String()) + require.Equal(t, "v0:\n2:+5:test%0A,\n7:+5:test%0A,\n0:+5:test%0A:\n6", newPatch.String()) } -func TestPatch_Undo(t *testing.T) { - patch, err := NewPatchFromString("v0:\n0:+5:test%0A") - require.Nil(t, err) - newPatch := patch.Undo() - require.Equal(t, 1, len(newPatch.Changes)) - require.Equal(t, "v0:\n0:-5:test%0A", newPatch.String()) - - patch, err = NewPatchFromString("v0:\n1:-5:test%0A") - require.Nil(t, err) - newPatch = patch.Undo() - require.Equal(t, 1, len(newPatch.Changes)) - require.Equal(t, "v0:\n1:+5:test%0A", newPatch.String()) - - patch, err = NewPatchFromString("v0:\n2:+5:test%0A") - require.Nil(t, err) - newPatch = patch.Undo() - require.Equal(t, 1, len(newPatch.Changes)) - require.Equal(t, "v0:\n2:-5:test%0A", newPatch.String()) - - patch, err = NewPatchFromString("v0:\n7:-5:test%0A") - require.Nil(t, err) - newPatch = patch.Undo() - require.Equal(t, 1, len(newPatch.Changes)) - require.Equal(t, "v0:\n7:+5:test%0A", newPatch.String()) - - patch, err = NewPatchFromString("v0:\n2:-5:test%0A,\n7:+5:test%0A") - require.Nil(t, err) - newPatch = patch.Undo() - require.Equal(t, 2, len(newPatch.Changes)) - require.Equal(t, "v0:\n7:-5:test%0A,\n2:+5:test%0A", newPatch.String()) - - patch, err = NewPatchFromString("v0:\n2:+5:test%0A,\n7:-5:test%0A,\n0:-5:test%0A") - require.Nil(t, err) - newPatch = patch.Undo() - require.Equal(t, 3, len(newPatch.Changes)) - require.Equal(t, "v0:\n0:+5:test%0A,\n7:+5:test%0A,\n2:-5:test%0A", newPatch.String()) -} - -func TestPatch_Transform(t *testing.T) { - // Test set 1 - patch1, err := NewPatchFromString("v1:\n0:-1:a") - require.Nil(t, err) - patch2, err := NewPatchFromString("v0:\n3:-8:deletion,\n3:+6:insert") - require.Nil(t, err) - newPatch := patch2.Transform([]*Patch{patch1}, true) - require.Equal(t, 2, len(newPatch.Changes)) - require.Equal(t, "v2:\n2:-8:deletion,\n2:+6:insert", newPatch.String()) - - // Test set 2 - patch1, err = NewPatchFromString("v1:\n0:-1:a") - require.Nil(t, err) - patch2, err = NewPatchFromString("v2:\n0:-1:b") - require.Nil(t, err) - patch3, err := NewPatchFromString("v0:\n3:-8:deletion,\n3:+6:insert") - require.Nil(t, err) - newPatch = patch3.Transform([]*Patch{patch1, patch2}, true) - require.Equal(t, 2, len(newPatch.Changes)) - require.Equal(t, "v3:\n1:-8:deletion,\n1:+6:insert", newPatch.String()) +func TestPatch_Simplify(t *testing.T) { + tests := []struct { + desc string + patchStr string + expected string + }{ + { + desc: "Double-Insert, Adjacent", + patchStr: "v1:\n0:+1:a,\n1:+1:b:\n10", + expected: "v1:\n0:+1:a,\n1:+1:b:\n10", + }, + { + desc: "Double-Remove, Adjacent", + patchStr: "v1:\n0:-1:a,\n1:-1:b:\n10", + expected: "v1:\n0:-2:ab:\n10", + }, + { + desc: "Insert-Remove, Adjacent", + patchStr: "v1:\n0:+1:a,\n1:-1:b:\n10", + expected: "v1:\n0:+1:a,\n1:-1:b:\n10", + }, + { + desc: "Remove-Insert, Adjacent", + patchStr: "v1:\n0:-1:a,\n1:+1:b:\n10", + expected: "v1:\n0:-1:a,\n1:+1:b:\n10", + }, + { + desc: "Double-Insert, Not adjacent", + patchStr: "v1:\n0:+1:a,\n2:+1:b:\n10", + expected: "v1:\n0:+1:a,\n2:+1:b:\n10", + }, + { + desc: "Double-Remove, Not adjacent", + patchStr: "v1:\n0:-1:a,\n2:-1:b:\n10", + expected: "v1:\n0:-1:a,\n2:-1:b:\n10", + }, + { + desc: "Insert-Remove, Not adjacent", + patchStr: "v1:\n0:+1:a,\n2:-1:b:\n10", + expected: "v1:\n0:+1:a,\n2:-1:b:\n10", + }, + { + desc: "Remove-Insert, Not adjacent", + patchStr: "v1:\n0:-1:a,\n2:+1:b:\n10", + expected: "v1:\n0:-1:a,\n2:+1:b:\n10", + }, + { + desc: "Triple-Insert, Adjacent", + patchStr: "v1:\n0:+1:a,\n1:+1:b,\n2:+1:c:\n10", + expected: "v1:\n0:+1:a,\n1:+1:b,\n2:+1:c:\n10", + }, + { + desc: "Triple-Remove, Adjacent", + patchStr: "v1:\n0:-1:a,\n1:-1:b,\n2:-1:c:\n10", + expected: "v1:\n0:-3:abc:\n10", + }, + { + desc: "Double-Insert, Single Remove, Adjacent", + patchStr: "v1:\n0:+1:a,\n1:+1:b,\n2:-1:c:\n10", + expected: "v1:\n0:+1:a,\n1:+1:b,\n2:-1:c:\n10", + }, + { + desc: "Single-Remove, Double-Insert, Adjacent", + patchStr: "v1:\n0:-1:a,\n1:+1:b,\n2:+1:c:\n10", + expected: "v1:\n0:-1:a,\n1:+1:b,\n2:+1:c:\n10", + }, + { + desc: "Double-Remove, Single Insert, Adjacent", + patchStr: "v1:\n0:-1:a,\n1:-1:b,\n2:+1:c:\n10", + expected: "v1:\n0:-2:ab,\n2:+1:c:\n10", + }, + { + desc: "Single-Insert, Double-Remove, Adjacent", + patchStr: "v1:\n0:+1:a,\n1:-1:b,\n2:-1:c:\n10", + expected: "v1:\n0:+1:a,\n1:-2:bc:\n10", + }, + { + desc: "Double-Insert, Single Remove, Not adjacent", + patchStr: "v1:\n0:+1:a,\n2:+1:b,\n3:-1:c:\n10", + expected: "v1:\n0:+1:a,\n2:+1:b,\n3:-1:c:\n10", + }, + { + desc: "Single-Remove, Double-Insert, Not adjacent", + patchStr: "v1:\n0:-1:a,\n1:+1:b,\n3:+1:c:\n10", + expected: "v1:\n0:-1:a,\n1:+1:b,\n3:+1:c:\n10", + }, + { + desc: "Double-Remove, Single Insert, Not adjacent", + patchStr: "v1:\n0:-1:a,\n2:-1:b,\n3:+1:c:\n10", + expected: "v1:\n0:-1:a,\n2:-1:b,\n3:+1:c:\n10", + }, + { + desc: "Single-Insert, Double-Remove, Not adjacent", + patchStr: "v1:\n0:+1:a,\n1:-1:b,\n3:-1:c:\n10", + expected: "v1:\n0:+1:a,\n1:-1:b,\n3:-1:c:\n10", + }, + { + desc: "Interleaved Insert-Delete-Insert, Adjacent", + patchStr: "v1:\n0:+1:a,\n1:-1:b,\n2:+1:c:\n10", + expected: "v1:\n0:+1:a,\n1:-1:b,\n2:+1:c:\n10", + }, + { + desc: "Interleaved Delete-Insert-Delete, Adjacent", + patchStr: "v1:\n0:-1:a,\n1:+1:b,\n2:-1:c:\n10", + expected: "v1:\n0:-1:a,\n1:+1:b,\n2:-1:c:\n10", + }, + } + + for _, test := range tests { + patch, err := NewPatchFromString(test.patchStr) + require.Nil(t, err) + + if want, got := test.expected, patch.String(); want != got { + t.Errorf("TestPatchSimplify[%s]: Expected %s, but got %s. Diffs: %v", test.desc, want, got, pretty.Diff(want, got)) + } + } } diff --git a/modules/patching/patcher.go b/modules/patching/patcher.go index a0dc918..3998bbf 100644 --- a/modules/patching/patcher.go +++ b/modules/patching/patcher.go @@ -4,8 +4,8 @@ import ( "bytes" "errors" "fmt" - - "github.com/CodeCollaborate/Server/utils" + "strings" + "unicode/utf8" ) // PatchTextFromString applies the provided patches onto the given text. The patches are applied strictly in the order given. @@ -23,46 +23,71 @@ func PatchTextFromString(text string, patchesStr []string) (string, error) { return PatchText(text, patches) } +// ErrorIllegalLocation is the error thrown if a diff attempts to insert in an invalid location, such as between an \r and \n +var ErrorIllegalLocation = errors.New("Attempted to apply diff at an illegal lcoation") + // PatchText applies the provided patches onto the given text. The patches are applied strictly in the order given. // This method completes in O(n*m) time, where n is the base text length, and m is the number of patches. func PatchText(text string, patches []*Patch) (string, error) { + useCRLF := strings.Contains(text, "\r\n") + for _, patch := range patches { + noOpLength := 0 + prevEndIndex := 0 + var prevDiff *Diff var buffer bytes.Buffer - startIndex := 0 + + if useCRLF { + patch.ConvertToCRLF(text) + } + for _, diff := range patch.Changes { - if startIndex < 0 || startIndex >= len(text) || diff.StartIndex < 0 { - utils.LogError("PatchText: Encountered invalid diff", errors.New("Slice out of bounds"), utils.LogFields{ - "Diff": diff, - "Patch": patch, - "Text": text, - }) - return "", fmt.Errorf("Invalid patch range: [%d, %d] for text length %d", startIndex, diff.StartIndex, len(text)) + if diff.StartIndex > 0 && diff.StartIndex < utf8.RuneCountInString(text) && + text[diff.StartIndex-1] == '\r' && text[diff.StartIndex] == '\n' { + return "", ErrorIllegalLocation + } + + noOpLength = diff.StartIndex + if prevDiff != nil { + if prevDiff.Insertion || prevDiff.StartIndex == diff.StartIndex { + noOpLength = diff.StartIndex - prevDiff.StartIndex + } else { + if prevDiff.StartIndex+prevDiff.Length() > diff.StartIndex { + return "", errors.New("Attempted to modify diff within range of previous deletion") + } + noOpLength = diff.StartIndex - (prevDiff.StartIndex + prevDiff.Length()) + } } - // Copy anything before the changes - if startIndex < diff.StartIndex { - buffer.WriteString(text[startIndex:diff.StartIndex]) + + // Copy any text that is untouched + if noOpLength > 0 { + buffer.WriteString(text[prevEndIndex : prevEndIndex+noOpLength]) } + if diff.Insertion { - // insert item + // Commit insertion buffer.WriteString(diff.Changes) - // If the diff's startIndex is greater, move it up. - // Otherwise, a previous delete may have deleted over the start index. - if startIndex < diff.StartIndex { - startIndex = diff.StartIndex - } + // End index is incremented only by the no-op length; + // insertions do not change the index in the original text + prevEndIndex += noOpLength } else { - // validate that we're deleting the right characters - if want, got := diff.Changes, text[diff.StartIndex:diff.StartIndex+diff.Length()]; want != got { - return "", fmt.Errorf("PatchText: Deleted text %q does not match changes in diff: %q", got, want) - } + // Move to start of deletion + prevEndIndex += noOpLength - // shift the start index of the next round - startIndex = diff.StartIndex + diff.Length() + if text[prevEndIndex:prevEndIndex+diff.Length()] != diff.Changes { + return "", fmt.Errorf("PatchText: Deleted text [%s] does not match changes in diff: [%s]", text[prevEndIndex:prevEndIndex+diff.Length()], diff.Changes) + } + // Skip past the text that is deleted + prevEndIndex += diff.Length() } + prevDiff = diff } + // Copy the remainder - buffer.WriteString(text[startIndex:]) + if prevEndIndex < len(text) { + buffer.WriteString(text[prevEndIndex:]) + } text = buffer.String() } diff --git a/modules/patching/patcher_test.go b/modules/patching/patcher_test.go index 9c2d9a9..ef39378 100644 --- a/modules/patching/patcher_test.go +++ b/modules/patching/patcher_test.go @@ -31,85 +31,85 @@ func TestFilePatcher_ApplyPatch(t *testing.T) { }{ { desc: "Single Patch, Single insertion", - patches: getPatchesOrDie(t, "v0:\n2:+1:a"), + patches: getPatchesOrDie(t, "v0:\n2:+1:a:\n10"), text: "test", expected: "teast", }, { desc: "Single Patch, Single deletion", - patches: getPatchesOrDie(t, "v0:\n2:-1:s"), + patches: getPatchesOrDie(t, "v0:\n2:-1:s:\n10"), text: "test", expected: "tet", }, { desc: "Single Patch, Single deletion, Incorrect base text", - patches: getPatchesOrDie(t, "v0:\n2:-1:s"), + patches: getPatchesOrDie(t, "v0:\n2:-1:s:\n10"), text: "aaaa", - error: "PatchText: Deleted text \"a\" does not match changes in diff: \"s\"", + error: "PatchText: Deleted text [a] does not match changes in diff: [s]", }, { desc: "Single Patch, Double insertion", - patches: getPatchesOrDie(t, "v0:\n2:+1:m,\n3:+2:ab"), + patches: getPatchesOrDie(t, "v0:\n2:+1:m,\n3:+2:ab:\n10"), text: "test", expected: "temsabt", }, { desc: "Single Patch, Double deletion", - patches: getPatchesOrDie(t, "v0:\n0:-1:t,\n2:-2:st"), + patches: getPatchesOrDie(t, "v0:\n0:-1:t,\n2:-2:st:\n10"), text: "test", expected: "e", }, { desc: "Single Patch, Insert+Delete", - patches: getPatchesOrDie(t, "v0:\n1:+1:z,\n2:-2:st"), + patches: getPatchesOrDie(t, "v0:\n1:+1:z,\n2:-2:st:\n10"), text: "test", expected: "tze", }, { desc: "Single Patch, Delete+Insert", - patches: getPatchesOrDie(t, "v0:\n1:-2:es,\n2:+2:lm"), + patches: getPatchesOrDie(t, "v0:\n1:-2:es,\n4:+2:lm:\n10"), text: "test", - expected: "tlmt", + expected: "ttlm", }, { desc: "Double Patch, Single Insertions, 1 first", - patches: getPatchesOrDie(t, "v0:\n1:+1:a", "v1:\n1:+1:b"), + patches: getPatchesOrDie(t, "v0:\n1:+1:a:\n10", "v1:\n1:+1:b:\n10"), text: "test", expected: "tbaest", }, { desc: "Double Patch, Single Insertions, 2 first", - patches: getPatchesOrDie(t, "v0:\n1:+1:a", "v1:\n2:+1:b"), + patches: getPatchesOrDie(t, "v0:\n1:+1:a:\n10", "v1:\n2:+1:b:\n10"), text: "test", expected: "tabest", }, { desc: "Double Patch, Single Deletions, 1 first", - patches: getPatchesOrDie(t, "v0:\n1:-1:e", "v1:\n1:-1:s"), + patches: getPatchesOrDie(t, "v0:\n1:-1:e:\n10", "v1:\n1:-1:s:\n10"), text: "test", expected: "tt", }, { desc: "Double Patch, Insert-Deletes", - patches: getPatchesOrDie(t, "v0:\n1:+1:z,\n2:-2:st", "v1:\n0:+2:aa,\n2:-1:e"), + patches: getPatchesOrDie(t, "v0:\n1:+1:z,\n2:-2:st:\n10", "v1:\n0:+2:aa,\n2:-1:e:\n10"), text: "test", expected: "aatz", }, { desc: "Double Patch, Delete-Inserts", - patches: getPatchesOrDie(t, "v0:\n1:-2:es,\n2:+2:lm", "v1:\n0:-2:tl,\n3:+2:kk"), + patches: getPatchesOrDie(t, "v0:\n1:-2:es,\n3:+2:lm:\n10", "v1:\n0:-2:tl,\n3:+2:kk:\n10"), text: "test", expected: "mkkt", }, { desc: "Double Patch, Insert-Delete, Delete-Insert", - patches: getPatchesOrDie(t, "v0:\n1:+1:z,\n2:-2:st", "v1:\n0:-2:tz,\n3:+2:ab"), + patches: getPatchesOrDie(t, "v0:\n1:+1:z,\n2:-2:st:\n10", "v1:\n0:-2:tz,\n3:+2:ab:\n10"), text: "test", expected: "eab", }, { desc: "Double Patch, Delete-Insert, Insert-Delete", - patches: getPatchesOrDie(t, "v0:\n1:-2:es,\n2:+2:lm", "v1:\n0:-2:tl,\n3:+2:kk"), + patches: getPatchesOrDie(t, "v0:\n1:-2:es,\n3:+2:lm:\n10", "v1:\n0:-2:tl,\n3:+2:kk:\n10"), text: "test", expected: "mkkt", }, @@ -132,9 +132,8 @@ func TestFilePatcher_ApplyPatch(t *testing.T) { continue } else { if want, got := test.expected, result; want != got { - t.Errorf("Expected %s, but got %s. Diffs: %v", want, got, pretty.Diff(want, got)) + t.Errorf("TestApplyPatch[%s]: Expected %s, but got %s. Diffs: %v", test.desc, want, got, pretty.Diff(want, got)) } } - } } diff --git a/modules/patching/transformer.go b/modules/patching/transformer.go new file mode 100644 index 0000000..2832c3e --- /dev/null +++ b/modules/patching/transformer.go @@ -0,0 +1,228 @@ +package patching + +import "errors" + +// TransformResult is a struct aggregating the results of the TransformPatches function +type TransformResult struct { + PatchXPrime *Patch + PatchYPrime *Patch +} + +// ErrorBaseDocumentLengthsDifferent is the error thrown when base document lengths for two patches are different +var ErrorBaseDocumentLengthsDifferent = errors.New("Base document lengths for patchX and patchY were different") + +// ErrorIllegalStateNoOpXYLen is the error thrown when we get into an invalid state (-1) +var ErrorIllegalStateNoOpXYLen = errors.New("Got to invalid state based on noOpXLen and noOpYLen") + +// TransformPatches takes two patches, and produces their component opposites (A -> A'), (B -> B') +func TransformPatches(patchX *Patch, patchY *Patch) (*TransformResult, error) { + if patchX.DocLength != patchY.DocLength { + return nil, ErrorBaseDocumentLengthsDifferent + } + + patchXPrime := Diffs{} + patchYPrime := Diffs{} + + indexX := -1 + indexY := -1 + + startIndexX := 0 + startIndexY := 0 + + noOpXLen := 0 + var diffX *Diff + noOpYLen := 0 + var diffY *Diff + + getNextDiffX := func() { + indexX++ + + if indexX < patchX.Changes.Len() { + diffX = patchX.Changes[indexX].clone() + + noOpXLen = diffX.StartIndex + if indexX > 0 { + prev := patchX.Changes[indexX-1] + if prev.Insertion || prev.StartIndex == diffX.StartIndex { + noOpXLen = diffX.StartIndex - prev.StartIndex + } else { + noOpXLen = diffX.StartIndex - (prev.StartIndex + prev.Length()) + } + } + } else { + // last no-op + if patchX.Changes.Len() != 0 { + prev := patchX.Changes[patchX.Changes.Len()-1] + noOpXLen = patchX.DocLength - prev.StartIndex + if !prev.Insertion { + noOpXLen -= prev.Length() + } + } else { + noOpXLen = patchX.DocLength + } + diffX = nil + } + } + + getNextDiffY := func() { + indexY++ + + if indexY < patchY.Changes.Len() { + diffY = patchY.Changes[indexY].clone() + + noOpYLen = diffY.StartIndex + if indexY > 0 { + prev := patchY.Changes[indexY-1] + if prev.Insertion || prev.StartIndex == diffY.StartIndex { + noOpYLen = diffY.StartIndex - prev.StartIndex + } else { + noOpYLen = diffY.StartIndex - (prev.StartIndex + prev.Length()) + } + } + } else { + // last no-op + if patchY.Changes.Len() != 0 { + prev := patchY.Changes[patchY.Changes.Len()-1] + noOpYLen = patchY.DocLength - prev.StartIndex + if !prev.Insertion { + noOpYLen -= prev.Length() + } + } else { + noOpYLen = patchY.DocLength + } + diffY = nil + } + } + + getNextDiffX() + getNextDiffY() + + for { + // Min of noOpXLen, noOpYLen + noOpLength := noOpXLen + if noOpYLen < noOpXLen { + noOpLength = noOpYLen + } + // Max of noOpLength, 0 + if noOpLength < 0 { + noOpLength = 0 + } + startIndexX += noOpLength + startIndexY += noOpLength + noOpXLen -= noOpLength + noOpYLen -= noOpLength + if diffX == nil && diffY == nil { + break + } + + diffXLen := noOpXLen + if diffX != nil { + diffXLen = diffX.Length() + } + diffYLen := noOpYLen + if diffY != nil { + diffYLen = diffY.Length() + } + + if diffX != nil && diffX.Insertion && noOpXLen == 0 { + patchXPrime = append(patchXPrime, NewDiff(true, startIndexX, diffX.Changes)) + + startIndexY += diffXLen + + getNextDiffX() + continue + } else if diffY != nil && diffY.Insertion && noOpYLen == 0 { + patchYPrime = append(patchYPrime, NewDiff(true, startIndexY, diffY.Changes)) + + startIndexX += diffYLen + + getNextDiffY() + continue + } + + if noOpXLen == 0 && noOpYLen == 0 { + if diffXLen < diffYLen { + // Nothing to commit + + // Already been deleted; remove first lenX characters from diffY + diffY.Changes = diffY.Changes[diffXLen:] + + // Done with this diffX + getNextDiffX() + } else if diffXLen == diffYLen { + // Both deleted the same text; nothing to commit + + getNextDiffX() + getNextDiffY() + } else { + // Nothing to commit + + // Already been deleted; remove first lenX characters from diffY + diffX.Changes = diffX.Changes[diffYLen:] + + // Done with this diffX + getNextDiffY() + } + } else if noOpXLen == 0 && noOpYLen > 0 { + commitLength := noOpYLen + if diffXLen < commitLength { + commitLength = diffXLen + } + + patchXPrime = append(patchXPrime, NewDiff(false, startIndexX, diffX.Changes[0:commitLength])) + + startIndexX += commitLength + diffX.Changes = diffX.Changes[commitLength:] + + noOpYLen -= commitLength + + // If we have exhausted the entire diffY, proceed to next diff + if diffX.Length() == 0 { + getNextDiffX() + } + } else if noOpXLen > 0 && noOpYLen == 0 { + commitLength := noOpXLen + if diffYLen < commitLength { + commitLength = diffYLen + } + + patchYPrime = append(patchYPrime, NewDiff(false, startIndexY, diffY.Changes[0:commitLength])) + + startIndexY += commitLength + diffY.Changes = diffY.Changes[commitLength:] + + noOpXLen -= commitLength + + // If we have exhausted the entire diffY, proceed to next diff + if diffY.Length() == 0 { + getNextDiffY() + } + } else { + return nil, ErrorIllegalStateNoOpXYLen + } + } + + // Less efficient, but simpler + newDocXLen := patchX.DocLength + for _, diff := range patchY.Changes { + if diff.Insertion { + newDocXLen += diff.Length() + } else { + newDocXLen -= diff.Length() + } + } + + newDocYLen := patchY.DocLength + for _, diff := range patchX.Changes { + if diff.Insertion { + newDocYLen += diff.Length() + } else { + newDocYLen -= diff.Length() + } + } + + return &TransformResult{ + NewPatch(patchY.BaseVersion+1, patchXPrime, newDocXLen), + NewPatch(patchX.BaseVersion+1, patchYPrime, newDocYLen), + }, nil +} diff --git a/modules/patching/transformer_test.go b/modules/patching/transformer_test.go new file mode 100644 index 0000000..dd6e1d8 --- /dev/null +++ b/modules/patching/transformer_test.go @@ -0,0 +1,527 @@ +package patching + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func getPatchOrDie(t *testing.T, patchStr string) *Patch { + patch, err := NewPatchFromString(patchStr) + require.Nil(t, err) + + return patch +} + +/** + * Created by wongb on 3/25/17. + */ +func TestPrecedence(t *testing.T) { + + baseText := "abdeletion" + patchA, err := NewPatchFromString("v0:\n2:-8:deletion,\n2:+6:insert:\n10") + require.Nil(t, err) + patchB, err := NewPatchFromString("v0:\n10:+1:a:\n10") + require.Nil(t, err) + + // Test with A having precedence + result, err := TransformPatches(patchA, patchB) + require.Nil(t, err) + // Validate the the document that each path produces is the same + resStringA, err := PatchText(baseText, []*Patch{patchA, result.PatchYPrime}) + require.Nil(t, err) + resStringB, err := PatchText(baseText, []*Patch{patchB, result.PatchXPrime}) + require.Nil(t, err) + require.Equal(t, resStringA, resStringB, "Resultant documents were different") + + // Test with B having precedence + result, err = TransformPatches(patchB, patchA) + require.Nil(t, err) + // Validate the the document that each path produces is the same + resStringA, err = PatchText(baseText, []*Patch{patchB, result.PatchYPrime}) + require.Nil(t, err) + resStringB, err = PatchText(baseText, []*Patch{patchA, result.PatchXPrime}) + require.Nil(t, err) + require.Equal(t, resStringA, resStringB, "Resultant documents were different") + +} + +func TestOverlappingDeletes(t *testing.T) { + baseText := "\n\tHello, my name is Ben. This is a test of whether this works properly.\n" + + "\t\n" + + "\tWow eclipse is dumb. It changed my \"Properly\" word to a entire public main method\n" + + "\ttesting\n" + + "\tSystem.out.println(\"Hellow this is a test\");\n" + + "\t\n" + + "\tif (true == false) {\n" + + "\t\tdo all the things\n" + + "\t\t2 = 1\n" + + "\t\tThis is definitely coherent. DEFINITELY\n" + + "\t}" + patchA, err := NewPatchFromString("v557:\n0:-309:%0A%09Hello%2C+my+name+is+Ben.+This+is+a+test+of+whether+this+works+properly.%0A%09%0A%09Wow+eclipse+is+dumb.+It+changed+my+%22Properly%22+word+to+a+entire+public+main+method%0A%09testing%0A%09System.out.println%28%22Hellow+this+is+a+test%22%29%3B%0A%09%0A%09if+%28true+%3D%3D+false%29+%7B%0A%09%09do+all+the+things%0A%09%092+%3D+1%0A%09%09This+is+definitely+coherent.++DEFINITELY%0A%09%7D,\n0:+1:m:\n309") + require.Nil(t, err) + patchB, err := NewPatchFromString("v557:\n308:-1:%7D:\n309") + require.Nil(t, err) + + // Test with A having precedence + result, err := TransformPatches(patchA, patchB) + require.Nil(t, err) + // Validate the the document that each path produces is the same + resStringA, err := PatchText(baseText, []*Patch{patchA, result.PatchYPrime}) + require.Nil(t, err) + resStringB, err := PatchText(baseText, []*Patch{patchB, result.PatchXPrime}) + require.Nil(t, err) + require.Equal(t, resStringA, resStringB, "Resultant documents were different") + + // Test with B having precedence + result, err = TransformPatches(patchB, patchA) + require.Nil(t, err) + // Validate the the document that each path produces is the same + resStringA, err = PatchText(baseText, []*Patch{patchB, result.PatchYPrime}) + require.Nil(t, err) + resStringB, err = PatchText(baseText, []*Patch{patchA, result.PatchXPrime}) + require.Nil(t, err) + require.Equal(t, resStringA, resStringB, "Resultant documents were different") +} + +type transformationTest struct { + desc string + patchA *Patch + patchB *Patch + baseText string + expectedPatchAPrime *Patch + expectedPatchBPrime *Patch + canReverse bool +} + +func runTests(t *testing.T, tests []transformationTest) { + for _, test := range tests { + + result, err := TransformPatches(test.patchA, test.patchB) + require.Nil(t, err) + + require.Equal(t, test.expectedPatchAPrime.String(), result.PatchXPrime.String(), "TestConsolidator[%s]: Patch A' was incorrect expected [%s], got [%s]", + test.desc, strings.Replace(test.expectedPatchAPrime.String(), "\n", "\\n", -1), strings.Replace(result.PatchXPrime.String(), "\n", "\\n", -1)) + require.Equal(t, test.expectedPatchBPrime.String(), result.PatchYPrime.String(), "TestConsolidator[%s]: Patch B' was incorrect expected [%s], got [%s]", + test.desc, strings.Replace(test.expectedPatchBPrime.String(), "\n", "\\n", -1), strings.Replace(result.PatchYPrime.String(), "\n", "\\n", -1)) + + // Validate the the document that each path produces is the same + resStringA, err := PatchText(test.baseText, []*Patch{test.patchA, result.PatchYPrime}) + require.Nil(t, err) + resStringB, err := PatchText(test.baseText, []*Patch{test.patchB, result.PatchXPrime}) + require.Nil(t, err) + require.Equal(t, resStringA, resStringB, "TestConsolidator[%s]: Document was different based on patch application order", test.desc) + + // If is reversible (does not require precedence, try running it in reverse + if test.canReverse { + result, err := TransformPatches(test.patchB, test.patchA) + require.Nil(t, err) + + require.Equal(t, test.expectedPatchBPrime.String(), result.PatchXPrime.String(), "TestConsolidator[%s-Reverse]: Patch B' was incorrect expected [%s], got [%s]", + test.desc, strings.Replace(test.expectedPatchBPrime.String(), "\n", "\\n", -1), strings.Replace(result.PatchXPrime.String(), "\n", "\\n", -1)) + require.Equal(t, test.expectedPatchAPrime.String(), result.PatchYPrime.String(), "TestConsolidator[%s-Reverse]: Patch A' was incorrect expected [%s], got [%s]", + test.desc, strings.Replace(test.expectedPatchAPrime.String(), "\n", "\\n", -1), strings.Replace(result.PatchYPrime.String(), "\n", "\\n", -1)) + + // Validate the the document that each path produces is the same + resStringA, err := PatchText(test.baseText, []*Patch{test.patchB, result.PatchYPrime}) + require.Nil(t, err) + resStringB, err := PatchText(test.baseText, []*Patch{test.patchA, result.PatchXPrime}) + require.Nil(t, err) + require.Equal(t, resStringA, resStringB, "TestConsolidator[%s-Reverse]: Document was different based on patch application order", test.desc) + } + } +} + +func TestEmptyDiffs(t *testing.T) { + tests := []transformationTest{ + { + "DiffA empty", + getPatchOrDie(t, "v1:\n:\n8"), + getPatchOrDie(t, "v1:\n6:+4:str2:\n8"), + "baseText", + getPatchOrDie(t, "v2:\n:\n12"), + getPatchOrDie(t, "v2:\n6:+4:str2:\n8"), + true, + }, { + + "DiffB empty", + getPatchOrDie(t, "v1:\n0:+4:str1:\n8"), + getPatchOrDie(t, "v1:\n:\n8"), + "baseText", + getPatchOrDie(t, "v2:\n0:+4:str1:\n8"), + getPatchOrDie(t, "v2:\n:\n12"), + true, + }, { + "Both empty", + getPatchOrDie(t, "v1:\n:\n8"), + getPatchOrDie(t, "v1:\n:\n8"), + "baseText", + getPatchOrDie(t, "v2:\n:\n8"), + getPatchOrDie(t, "v2:\n:\n8"), + true, + }, + } + runTests(t, tests) +} + +func TestTransform1A(t *testing.T) { + tests := []transformationTest{ + { + "Non-Overlapping strings", + getPatchOrDie(t, "v1:\n0:+4:str1:\n8"), + getPatchOrDie(t, "v1:\n6:+4:str2:\n8"), + "baseText", + getPatchOrDie(t, "v2:\n0:+4:str1:\n12"), + getPatchOrDie(t, "v2:\n10:+4:str2:\n12"), + true, + }, { + "Overlapping strings", + getPatchOrDie(t, "v1:\n2:+4:str1:\n8"), + getPatchOrDie(t, "v1:\n4:+4:str2:\n8"), + "baseText", + getPatchOrDie(t, "v2:\n2:+4:str1:\n12"), + getPatchOrDie(t, "v2:\n8:+4:str2:\n12"), + true, + }, + } + runTests(t, tests) +} + +func TestTransform1B(t *testing.T) { + tests := []transformationTest{ + { + "Non-Overlapping strings", + getPatchOrDie(t, "v1:\n0:+2:s1:\n8"), + getPatchOrDie(t, "v1:\n4:-4:Text:\n8"), + "baseText", + getPatchOrDie(t, "v2:\n0:+2:s1:\n4"), + getPatchOrDie(t, "v2:\n6:-4:Text:\n10"), + true, + }, + { + "Overlapping strings", + getPatchOrDie(t, "v1:\n2:+4:str1:\n8"), + getPatchOrDie(t, "v1:\n4:-4:Text:\n8"), + "baseText", + getPatchOrDie(t, "v2:\n2:+4:str1:\n4"), + getPatchOrDie(t, "v2:\n8:-4:Text:\n12"), + true, + }, + } + + runTests(t, tests) +} + +func TestTransform1C(t *testing.T) { + tests := []transformationTest{ + { + "Non-Overlapping strings", + getPatchOrDie(t, "v1:\n0:-2:ba:\n8"), + getPatchOrDie(t, "v1:\n4:+4:abcd:\n8"), + "baseText", + getPatchOrDie(t, "v2:\n0:-2:ba:\n12"), + getPatchOrDie(t, "v2:\n2:+4:abcd:\n6"), + true, + }, + { + "Overlapping strings", + getPatchOrDie(t, "v1:\n2:-4:seTe:\n8"), + getPatchOrDie(t, "v1:\n4:+4:abcd:\n8"), + "baseText", + getPatchOrDie(t, "v2:\n2:-2:se,\n8:-2:Te:\n12"), + getPatchOrDie(t, "v2:\n2:+4:abcd:\n4"), + true, + }, + } + + runTests(t, tests) +} + +func TestTransform1D(t *testing.T) { + tests := []transformationTest{ + { + "Non-Overlapping strings", + getPatchOrDie(t, "v1:\n2:-4:str1:\n16"), + getPatchOrDie(t, "v1:\n8:-4:str2:\n16"), + "bastr1sestr2Text", + getPatchOrDie(t, "v2:\n2:-4:str1:\n12"), + getPatchOrDie(t, "v2:\n4:-4:str2:\n12"), + true, + }, + { + "Non-Overlapping strings, adjacent", + getPatchOrDie(t, "v1:\n2:-4:str1:\n16"), + getPatchOrDie(t, "v1:\n6:-4:str2:\n16"), + "bastr1str2seText", + getPatchOrDie(t, "v2:\n2:-4:str1:\n12"), + getPatchOrDie(t, "v2:\n2:-4:str2:\n12"), + true, + }, + { + "Overlapping strings", + getPatchOrDie(t, "v1:\n2:-4:seTe:\n8"), + getPatchOrDie(t, "v1:\n4:-4:Text:\n8"), + "baseText", + getPatchOrDie(t, "v2:\n2:-2:se:\n4"), + getPatchOrDie(t, "v2:\n2:-2:xt:\n4"), + true, + }, + { + "Overlapping strings, B subset of A", + getPatchOrDie(t, "v1:\n2:-6:seText:\n8"), + getPatchOrDie(t, "v1:\n4:-2:Te:\n8"), + "baseText", + getPatchOrDie(t, "v2:\n2:-4:sext:\n6"), + getPatchOrDie(t, "v2:\n:\n2"), + true, + }, + } + runTests(t, tests) +} + +func TestTransform2A(t *testing.T) { + tests := []transformationTest{ + { + "Same length strings", + getPatchOrDie(t, "v1:\n4:+4:str1:\n8"), + getPatchOrDie(t, "v1:\n4:+4:str2:\n8"), + "testText", + getPatchOrDie(t, "v2:\n4:+4:str1:\n12"), + getPatchOrDie(t, "v2:\n8:+4:str2:\n12"), + false, + }, + { + "A longer", + getPatchOrDie(t, "v1:\n4:+8:longstr1:\n8"), + getPatchOrDie(t, "v1:\n4:+4:str2:\n8"), + "testText", + getPatchOrDie(t, "v2:\n4:+8:longstr1:\n12"), + getPatchOrDie(t, "v2:\n12:+4:str2:\n16"), + false, + }, + { + "B longer", + getPatchOrDie(t, "v1:\n4:+4:str1:\n8"), + getPatchOrDie(t, "v1:\n4:+8:longstr2:\n8"), + "testText", + getPatchOrDie(t, "v2:\n4:+4:str1:\n16"), + getPatchOrDie(t, "v2:\n8:+8:longstr2:\n12"), + false, + }, + } + runTests(t, tests) +} + +func TestTransform2B(t *testing.T) { + tests := []transformationTest{ + { + "Same length strings", + getPatchOrDie(t, "v1:\n2:+4:str1:\n8"), + getPatchOrDie(t, "v1:\n2:-4:stTe:\n8"), + "testText", + getPatchOrDie(t, "v2:\n2:+4:str1:\n4"), + getPatchOrDie(t, "v2:\n6:-4:stTe:\n12"), + true, + }, + { + "A longer", + getPatchOrDie(t, "v1:\n2:+8:longstr1:\n8"), + getPatchOrDie(t, "v1:\n2:-4:stTe:\n8"), + "testText", + getPatchOrDie(t, "v2:\n2:+8:longstr1:\n4"), + getPatchOrDie(t, "v2:\n10:-4:stTe:\n16"), + true, + }, + { + "B longer", + getPatchOrDie(t, "v1:\n4:+4:str1:\n14"), + getPatchOrDie(t, "v1:\n4:-6:longer:\n14"), + "testlongerText", + getPatchOrDie(t, "v2:\n4:+4:str1:\n8"), + getPatchOrDie(t, "v2:\n8:-6:longer:\n18"), + true, + }, + } + runTests(t, tests) +} + +func TestTransform2C(t *testing.T) { + tests := []transformationTest{ + { + "Same length strings", + getPatchOrDie(t, "v1:\n2:-4:stTe:\n8"), + getPatchOrDie(t, "v1:\n2:+4:str2:\n8"), + "testText", + getPatchOrDie(t, "v2:\n6:-4:stTe:\n12"), + getPatchOrDie(t, "v2:\n2:+4:str2:\n4"), + true, + }, + { + "A longer", + getPatchOrDie(t, "v1:\n4:-6:longer:\n14"), + getPatchOrDie(t, "v1:\n4:+4:str2:\n14"), + "testlongerText", + getPatchOrDie(t, "v2:\n8:-6:longer:\n18"), + getPatchOrDie(t, "v2:\n4:+4:str2:\n8"), + true, + }, + { + "B longer", + getPatchOrDie(t, "v1:\n2:-4:stTe:\n8"), + getPatchOrDie(t, "v1:\n2:+8:longstr2:\n8"), + "testText", + getPatchOrDie(t, "v2:\n10:-4:stTe:\n16"), + getPatchOrDie(t, "v2:\n2:+8:longstr2:\n4"), + true, + }, + } + runTests(t, tests) +} + +func TestTransform2D(t *testing.T) { + tests := []transformationTest{ + { + "Same length strings", + getPatchOrDie(t, "v1:\n2:-4:stTe:\n8"), + getPatchOrDie(t, "v1:\n2:-4:stTe:\n8"), + "testText", + getPatchOrDie(t, "v2:\n:\n4"), + getPatchOrDie(t, "v2:\n:\n4"), + true, + }, + { + "A longer", + getPatchOrDie(t, "v1:\n4:-6:longer:\n14"), + getPatchOrDie(t, "v1:\n4:-4:long:\n14"), + "testlongerText", + getPatchOrDie(t, "v2:\n4:-2:er:\n10"), + getPatchOrDie(t, "v2:\n:\n8"), + true, + }, + { + "B longer", + getPatchOrDie(t, "v1:\n2:-4:stlo:\n14"), + getPatchOrDie(t, "v1:\n2:-6:stlong:\n14"), + "testlongerText", + getPatchOrDie(t, "v2:\n:\n8"), + getPatchOrDie(t, "v2:\n2:-2:ng:\n10"), + true, + }, + } + runTests(t, tests) +} + +func TestTransform3A(t *testing.T) { + tests := []transformationTest{ + { + "Overlapping strings", + getPatchOrDie(t, "v1:\n5:+4:str1:\n8"), + getPatchOrDie(t, "v1:\n4:+4:str2:\n8"), + "testText", + getPatchOrDie(t, "v2:\n9:+4:str1:\n12"), + getPatchOrDie(t, "v2:\n4:+4:str2:\n12"), + true, + }, + { + "Non-overlapping strings", + getPatchOrDie(t, "v1:\n4:+4:str1:\n8"), + getPatchOrDie(t, "v1:\n0:+15:longTestString2:\n8"), + "testText", + getPatchOrDie(t, "v2:\n19:+4:str1:\n23"), + getPatchOrDie(t, "v2:\n0:+15:longTestString2:\n12"), + true, + }, + } + runTests(t, tests) +} + +func TestTransform3B(t *testing.T) { + tests := []transformationTest{ + { + "Overlapping strings", + getPatchOrDie(t, "v1:\n5:+4:str1:\n8"), + getPatchOrDie(t, "v1:\n4:-4:Text:\n8"), + "testText", + getPatchOrDie(t, "v2:\n4:+4:str1:\n4"), + getPatchOrDie(t, "v2:\n4:-1:T,\n9:-3:ext:\n12"), + true, + }, + { + "Non-overlapping strings", + getPatchOrDie(t, "v1:\n5:+4:str1:\n8"), + getPatchOrDie(t, "v1:\n2:-2:st:\n8"), + "testText", + getPatchOrDie(t, "v2:\n3:+4:str1:\n6"), + getPatchOrDie(t, "v2:\n2:-2:st:\n12"), + true, + }, + } + runTests(t, tests) +} + +func TestTransform3C(t *testing.T) { + tests := []transformationTest{ + { + "Overlapping strings", + getPatchOrDie(t, "v1:\n5:-3:ext:\n8"), + getPatchOrDie(t, "v1:\n4:+4:str2:\n8"), + "testText", + getPatchOrDie(t, "v2:\n9:-3:ext:\n12"), + getPatchOrDie(t, "v2:\n4:+4:str2:\n5"), + true, + }, + { + "Non-overlapping strings", + getPatchOrDie(t, "v1:\n5:-3:ext:\n8"), + getPatchOrDie(t, "v1:\n1:+2:s2:\n8"), + "testText", + getPatchOrDie(t, "v2:\n7:-3:ext:\n10"), + getPatchOrDie(t, "v2:\n1:+2:s2:\n5"), + true, + }, + } + runTests(t, tests) +} + +func TestTransform3D(t *testing.T) { + tests := []transformationTest{ + { + "Overlapping strings, A extends past B", + getPatchOrDie(t, "v1:\n4:-8:LongerTe:\n14"), + getPatchOrDie(t, "v1:\n2:-4:stLo:\n14"), + "testLongerText", + getPatchOrDie(t, "v2:\n2:-6:ngerTe:\n10"), + getPatchOrDie(t, "v2:\n2:-2:st:\n6"), + true, + }, + { + "Overlapping strings, A ends at same index as B", + getPatchOrDie(t, "v1:\n4:-4:Long:\n14"), + getPatchOrDie(t, "v1:\n2:-6:stLong:\n14"), + "testLongerText", + getPatchOrDie(t, "v2:\n:\n8"), + getPatchOrDie(t, "v2:\n2:-2:st:\n10"), + true, + }, + { + "Overlapping strings, A ends before B", + getPatchOrDie(t, "v1:\n4:-3:Lon:\n14"), + getPatchOrDie(t, "v1:\n2:-6:stLong:\n14"), + "testLongerText", + getPatchOrDie(t, "v2:\n:\n8"), + getPatchOrDie(t, "v2:\n2:-3:stg:\n11"), + true, + }, + { + "Non-overlapping strings", + getPatchOrDie(t, "v1:\n5:-3:ext:\n8"), + getPatchOrDie(t, "v1:\n1:-2:es:\n8"), + "testText", + getPatchOrDie(t, "v2:\n3:-3:ext:\n6"), + getPatchOrDie(t, "v2:\n1:-2:es:\n5"), + true, + }, + } + runTests(t, tests) +} diff --git a/modules/rabbitmq/rabbitCommandHandlers.go b/modules/rabbitmq/rabbitCommandHandlers.go index cfaaf35..6a6d548 100644 --- a/modules/rabbitmq/rabbitCommandHandlers.go +++ b/modules/rabbitmq/rabbitCommandHandlers.go @@ -4,15 +4,16 @@ import ( "encoding/json" "errors" + "github.com/gorilla/websocket" + "github.com/CodeCollaborate/Server/modules/datahandling/messages" "github.com/CodeCollaborate/Server/utils" - "github.com/gorilla/websocket" ) // RabbitCommandHandler handles all rabbit commands (sub/unsub) type RabbitCommandHandler struct { WSConn *websocket.Conn - WSID uint64 + QueueName string ExchangeName string } @@ -52,7 +53,7 @@ func (r RabbitCommandHandler) handleSubscribe(cmd RabbitCommandJSON) error { } msg := messages.NewEmptyResponse(messages.StatusSuccess, cmd.Tag) - err = BindQueue(ch, RabbitWebsocketQueueName(r.WSID), data.Key, r.ExchangeName) + err = BindQueue(ch, r.QueueName, data.Key, r.ExchangeName) if err != nil { msg = messages.NewEmptyResponse(messages.StatusFail, cmd.Tag) } @@ -84,7 +85,7 @@ func (r RabbitCommandHandler) handleUnsubscribe(cmd RabbitCommandJSON) error { } msg := messages.NewEmptyResponse(messages.StatusSuccess, cmd.Tag) - err = UnbindQueue(ch, RabbitWebsocketQueueName(r.WSID), data.Key, r.ExchangeName) + err = UnbindQueue(ch, r.QueueName, data.Key, r.ExchangeName) if err != nil { msg = messages.NewEmptyResponse(messages.StatusFail, cmd.Tag) } diff --git a/modules/rabbitmq/rabbitconfig.go b/modules/rabbitmq/rabbitconfig.go index ac63a2a..8d821f7 100644 --- a/modules/rabbitmq/rabbitconfig.go +++ b/modules/rabbitmq/rabbitconfig.go @@ -59,24 +59,20 @@ func NewAMQPPubSubCfg(exchangeName string, pubCfg *AMQPPubCfg, subCfg *AMQPSubCf // AMQPSubCfg represents the settings needed to create a new subscriber, including the queues and key bindings type AMQPSubCfg struct { - QueueID uint64 + QueueName string Keys []string IsWorkQueue bool + PrefetchCount int HandleMessageFunc func(AMQPMessage) error } -// QueueName generates the Queue -func (cfg AMQPSubCfg) QueueName() string { - return RabbitWebsocketQueueName(cfg.QueueID) -} - // RabbitUserQueueName returns the name of the Queue a websocket for the given user would have func RabbitUserQueueName(username string) string { return fmt.Sprintf("User-%s", username) } -// RabbitWebsocketQueueName returns the name of the Queue a websocket with the given ID would have -func RabbitWebsocketQueueName(queueID uint64) string { +// LocalWebsocketName returns the name of the Queue a websocket with the given ID would have +func LocalWebsocketName(queueID uint64) string { return fmt.Sprintf("WS-%s-%d", hostname, queueID) } @@ -107,6 +103,8 @@ type AMQPMessage struct { Persistent bool Message []byte ErrHandler func() + Ack func() error + Nack func() error } const ( @@ -115,4 +113,7 @@ const ( // ContentTypeCmd is the command content-type for an AMQPMessage ContentTypeCmd + + // ContentTypeWork is the work content-type for an AMQPMessage + ContentTypeWork ) diff --git a/modules/rabbitmq/rabbitconfig_test.go b/modules/rabbitmq/rabbitconfig_test.go index 5d20290..bfdb40d 100644 --- a/modules/rabbitmq/rabbitconfig_test.go +++ b/modules/rabbitmq/rabbitconfig_test.go @@ -54,14 +54,14 @@ func TestQueueName(t *testing.T) { queueID := i queueCfg := AMQPSubCfg{ - QueueID: queueID, + QueueName: LocalWebsocketName(queueID), Keys: []string{"Key1", "Key2"}, IsWorkQueue: false, } expected := "WS-" + hostname + "-" + strconv.FormatUint(queueID, 10) - if queueCfg.QueueName() != expected { - t.Fatalf("QueueName incorrect; expected [%s], got [%s]", expected, queueCfg.QueueName()) + if queueCfg.QueueName != expected { + t.Fatalf("QueueName incorrect; expected [%s], got [%s]", expected, queueCfg.QueueName) } } } diff --git a/modules/rabbitmq/rabbitmanager.go b/modules/rabbitmq/rabbitmanager.go index 509d308..b119a04 100644 --- a/modules/rabbitmq/rabbitmanager.go +++ b/modules/rabbitmq/rabbitmanager.go @@ -8,9 +8,10 @@ import ( "sync" "time" - "github.com/CodeCollaborate/Server/utils" "github.com/kr/pretty" "github.com/streadway/amqp" + + "github.com/CodeCollaborate/Server/utils" ) /** @@ -207,22 +208,33 @@ func RunSubscriber(cfg *AMQPPubSubCfg) error { defer ch.Close() _, err = ch.QueueDeclare( - cfg.SubCfg.QueueName(), // name (routing key) - cfg.SubCfg.IsWorkQueue, // durable - persist data upon restarts? - !cfg.SubCfg.IsWorkQueue, // delete when unused - no more clients attached - !cfg.SubCfg.IsWorkQueue, // exclusive - can only be used by this channel - false, // no-wait - do not wait for server to confirm that the queue has been created - nil, // arguments + cfg.SubCfg.QueueName, // name (routing key) + false, // durable - persist data upon restarts? + true, // delete when unused - no more clients attached + false, // exclusive - can only be used by this channel + false, // no-wait - do not wait for server to confirm that the queue has been created + nil, // arguments ) if err != nil { return err } - for _, key := range append(cfg.SubCfg.Keys, cfg.SubCfg.QueueName()) { + if cfg.SubCfg.PrefetchCount != 0 { + err = ch.Qos( + cfg.SubCfg.PrefetchCount, // prefetch count + 0, // prefetch size + false, // global + ) + if err != nil { + return err + } + } + + for _, key := range append(cfg.SubCfg.Keys, cfg.SubCfg.QueueName) { err = BindQueue(ch, - cfg.SubCfg.QueueName(), // queue name - key, // routing key - cfg.ExchangeName, // exchange + cfg.SubCfg.QueueName, // queue name + key, // routing key + cfg.ExchangeName, // exchange ) if err != nil { return err @@ -230,13 +242,13 @@ func RunSubscriber(cfg *AMQPPubSubCfg) error { } msgs, err := ch.Consume( - cfg.SubCfg.QueueName(), // queue - "", // consumer - true, // auto ack - false, // exclusive - false, // no local - false, // no wait - nil, // args + cfg.SubCfg.QueueName, // queue + "", // consumer + false, // auto-ack FIXME: is this right? + false, // exclusive + false, // no local + false, // no wait + nil, // args ) if err != nil { return err @@ -261,7 +273,21 @@ func RunSubscriber(cfg *AMQPPubSubCfg) error { RoutingKey: msg.RoutingKey, ContentType: contentType, Message: msg.Body, - Persistent: (msg.DeliveryMode == 2), + Persistent: msg.DeliveryMode == 2, + Ack: func() error { + return msg.Ack(false) + }, + Nack: func() error { + utils.LogDebug("Nacking message for unknown reasons", utils.LogFields{ + "RabbitMQ tag": msg.DeliveryTag, + "Content type": contentType, + "RoutingKey": msg.RoutingKey, + }) + return msg.Nack(false, true) + }, + ErrHandler: func() { + // do nothing (for now?) + }, } err = cfg.SubCfg.HandleMessageFunc(message) diff --git a/modules/rabbitmq/rabbitmanager_test.go b/modules/rabbitmq/rabbitmanager_test.go index c935216..2e71f54 100644 --- a/modules/rabbitmq/rabbitmanager_test.go +++ b/modules/rabbitmq/rabbitmanager_test.go @@ -3,15 +3,16 @@ package rabbitmq import ( "encoding/json" "errors" - "fmt" "reflect" "sync" "testing" "time" + "github.com/streadway/amqp" + "github.com/stretchr/testify/assert" + "github.com/CodeCollaborate/Server/modules/config" "github.com/CodeCollaborate/Server/utils" - "github.com/streadway/amqp" ) var testExchange = AMQPExchCfg{ @@ -135,7 +136,7 @@ func TestSendMessage(t *testing.T) { } queueID := uint64(0) - routingKey := fmt.Sprintf("WS-%s-%d", hostname, queueID) + routingKey := LocalWebsocketName(queueID) doneTesting := make(chan bool, 1) defer close(doneTesting) @@ -156,14 +157,26 @@ func TestSendMessage(t *testing.T) { pubSubCfg := &AMQPPubSubCfg{ ExchangeName: testExchange.ExchangeName, SubCfg: &AMQPSubCfg{ - QueueID: queueID, + QueueName: LocalWebsocketName(queueID), Keys: []string{}, IsWorkQueue: false, HandleMessageFunc: func(msg AMQPMessage) error { pubSubCtrl.Shutdown() - if !reflect.DeepEqual(msg, TestMessage) { - t.Fatal("Sent message does not equal received message") - } + + assert.NotNil(t, msg.Ack, "Message does not have Ack") + assert.NotNil(t, msg.Nack, "Message does not have Nack") + assert.NotNil(t, msg.ErrHandler, "Message does not have Nack") + + err := msg.Ack() + assert.Nil(t, err) + + // the messages aren't EXACT copies of each other b/c Nack, Ack, and ErrHandler are nil in TestMessage + // so we need to fix this + msg.Ack = nil + msg.Nack = nil + msg.ErrHandler = nil + + assert.EqualValues(t, TestMessage, msg, "Sent message does not equal received message") doneTesting <- true return nil }, @@ -246,7 +259,7 @@ func TestSubscription(t *testing.T) { "Header2": "Value2", "Header3": "Value3", }, - RoutingKey: RabbitWebsocketQueueName(queueID), + RoutingKey: LocalWebsocketName(queueID), ContentType: ContentTypeCmd, Persistent: false, Message: msgJSON, @@ -259,7 +272,7 @@ func TestSubscription(t *testing.T) { pubSubCfg := &AMQPPubSubCfg{ ExchangeName: testExchange.ExchangeName, SubCfg: &AMQPSubCfg{ - QueueID: queueID, + QueueName: LocalWebsocketName(queueID), Keys: []string{}, IsWorkQueue: false, HandleMessageFunc: func(msg AMQPMessage) error { @@ -269,8 +282,12 @@ func TestSubscription(t *testing.T) { rch := RabbitCommandHandler{ ExchangeName: testExchange.ExchangeName, WSConn: nil, - WSID: queueID, + QueueName: LocalWebsocketName(queueID), } + + err := msg.Ack() + assert.Nil(t, err) + return rch.HandleCommand(msg) default: t.Fatalf("Unexpected message type: %d", msg.ContentType) @@ -303,6 +320,20 @@ func TestSubscription(t *testing.T) { switch msg.ContentType { case ContentTypeMsg: pubSubCtrl.Shutdown() + + assert.NotNil(t, msg.Ack, "Message does not have Ack") + assert.NotNil(t, msg.Nack, "Message does not have Nack") + assert.NotNil(t, msg.ErrHandler, "Message does not have Nack") + + err := msg.Ack() + assert.Nil(t, err) + + // the messages aren't EXACT copies of each other b/c Nack, Ack, and ErrHandler are nil in TestMessage + // so we need to fix this + msg.Ack = nil + msg.Nack = nil + msg.ErrHandler = nil + if !reflect.DeepEqual(msg, TestMessage) { t.Fatal("Sent message does not equal received message") } diff --git a/runner.go b/runner.go index 0ee444e..953f55c 100644 --- a/runner.go +++ b/runner.go @@ -1,18 +1,20 @@ package main import ( + "crypto/tls" "flag" + "fmt" "net/http" _ "net/http/pprof" "os" - - "fmt" + "runtime" "github.com/CodeCollaborate/Server/modules/config" "github.com/CodeCollaborate/Server/modules/dbfs" "github.com/CodeCollaborate/Server/modules/handlers" "github.com/CodeCollaborate/Server/modules/rabbitmq" "github.com/CodeCollaborate/Server/utils" + "golang.org/x/crypto/acme/autocert" ) /** @@ -21,6 +23,9 @@ import ( var logDir = flag.String("log_dir", "./data/logs/", "log file location") +// note that runtime.NumCPU() is set to `runtime.GOMAXPROCS` by default +var workerPrefetch = flag.Int("worker_prefetch", runtime.NumCPU(), "number of entries that should be prefetched from RabbitMQ") + func main() { flag.Parse() @@ -31,6 +36,17 @@ func main() { } cfg := config.GetConfig() + go func() { + // enable profiling to `:(port)/debug/pprof/` + addr := fmt.Sprintf("0.0.0.0:%d", cfg.ServerConfig.Port+1) + err := http.ListenAndServe(addr, nil) + if err != nil { + utils.LogError("Failed to start pprof", err, utils.LogFields{ + "Address": addr, + }) + } + }() + // Get working directory dir, err := os.Getwd() utils.LogFatal("Could not get working directory", err, nil) @@ -42,6 +58,11 @@ func main() { // Creates a NewControl block for multithreading control AMQPControl := utils.NewControl(1) + // Kill the SetupRabbitExchange thread (Multithreading control) + defer func() { + AMQPControl.Exit <- true + }() + // RabbitMQ uses "Exchanges" as containers for Queues, and ours is initialized here. rabbitmq.SetupRabbitExchange( &rabbitmq.AMQPConnCfg{ @@ -56,19 +77,21 @@ func main() { }, ) - dbfs.Dbfs = new(dbfs.DatabaseImpl) + dbfsImpl := new(dbfs.DatabaseImpl) + handlers.StartWorker(dbfsImpl, *workerPrefetch) http.HandleFunc("/ws/", handlers.NewWSConn) - addr := fmt.Sprintf("0.0.0.0:%d", cfg.ServerConfig.Port) + addr := fmt.Sprintf(":%d", cfg.ServerConfig.Port) - _, certErr := os.Stat("config/TLS/cert.pem") - _, keyErr := os.Stat("config/TLS/key.pem") + //_, certErr := os.Stat("config/TLS/cert.pem") + //_, keyErr := os.Stat("config/TLS/key.pem") - useTLS := certErr == nil && keyErr == nil + //useTLS := certErr == nil && keyErr == nil utils.LogInfo("Starting server", utils.LogFields{ "Address": addr, - "TLS": useTLS, + "Host": cfg.ServerConfig.Host, + "TLS": cfg.ServerConfig.UseTLS, }) go func() { @@ -78,20 +101,32 @@ func main() { utils.LogError("Failed to start pprof", err, utils.LogFields{ "Address": addr, }) + } else { + utils.LogError("pprof debugging server started at 0.0.0.0:8000", err, utils.LogFields{ + "Address": addr, + }) } }() - if useTLS { - err = http.ListenAndServeTLS(addr, "config/TLS/cert.pem", "config/TLS/key.pem", nil) + if cfg.ServerConfig.UseTLS { + dirCache := autocert.DirCache("certs") + certManager := autocert.Manager{ + Prompt: autocert.AcceptTOS, + HostPolicy: autocert.HostWhitelist(cfg.ServerConfig.Host), //your domain here + Cache: dirCache, //folder for storing certificates + } + + server := &http.Server{ + Addr: addr, + TLSConfig: &tls.Config{ + GetCertificate: certManager.GetCertificate, + }, + } + + server.ListenAndServeTLS("", "") //key and cert are comming from Let's Encrypt } else { - utils.LogWarn("No Cert/Key pair found; starting without TLS", nil) err = http.ListenAndServe(addr, nil) } utils.LogError("Could not bind to port", err, nil) - - // Kill the SetupRabbitExchange thread (Multithreading control) - defer func() { - AMQPControl.Exit <- true - }() } diff --git a/scripts/server/update_server.sh b/scripts/server/update_server.sh index 1576600..cb1b617 100755 --- a/scripts/server/update_server.sh +++ b/scripts/server/update_server.sh @@ -8,14 +8,6 @@ else printf "%-10s\n" "FAIL" fi -#printf "%-60s" "Stopping CodeCollaborate daemon" -#sudo systemctl stop CodeCollaborate.service -#if [ $? -eq 0 ]; then -# printf "%-10s\n" "OK" -#else -# printf "%-10s\n" "FAIL" -#fi - printf "%-60s" "Building server binary" go build -o CodeCollaborateServer if [ $? -eq 0 ]; then @@ -38,6 +30,13 @@ else printf "\t%-10s\n" "CHMOD FAIL" fi +sudo setcap 'cap_net_bind_service=+ep' /CodeCollaborate/Server +if [ $? -eq 0 ]; then + printf "\t%-10s\n" "SETCAP OK" +else + printf "\t%-10s\n" "SETCAP FAIL" +fi + printf "%-60s" "Restarting systemctl service" sudo systemctl restart CodeCollaborate.service if [ $? -eq 0 ]; then diff --git a/utils/consolehook.go b/utils/consolehook.go index e055138..a0edee9 100644 --- a/utils/consolehook.go +++ b/utils/consolehook.go @@ -13,6 +13,8 @@ func MakeConsoleHook() *ConsoleHook { consoleLog.Out = os.Stdout // text formatter w/ color if it's supported consoleLog.Formatter = &logrus.TextFormatter{} + // Set debug level of this console logger + consoleLog.Level = logrus.DebugLevel return &ConsoleHook{logger: consoleLog} } @@ -26,7 +28,7 @@ func (hook *ConsoleHook) Fire(entry *logrus.Entry) error { if entry.Logger.Level >= entry.Level { switch entry.Level { case logrus.DebugLevel: - hook.logger.WithFields(entry.Data).Debug(entry.Data) + hook.logger.WithFields(entry.Data).Debug(entry.Message) case logrus.InfoLevel: hook.logger.WithFields(entry.Data).Info(entry.Message) case logrus.WarnLevel: @@ -36,7 +38,7 @@ func (hook *ConsoleHook) Fire(entry *logrus.Entry) error { case logrus.FatalLevel: hook.logger.WithFields(entry.Data).Fatal(entry.Message) case logrus.PanicLevel: - hook.logger.WithFields(entry.Data).Panic(entry.Level) + hook.logger.WithFields(entry.Data).Panic(entry.Message) } } return nil diff --git a/utils/control.go b/utils/control.go index a2969dc..c364d38 100644 --- a/utils/control.go +++ b/utils/control.go @@ -9,19 +9,31 @@ import "sync" // Control groups common multi-threading control variables, allowing for waiting until thread is ready, // and setting exit flag. type Control struct { + sync.Mutex Ready sync.WaitGroup Exit chan bool shutdown sync.Once + exited bool } // Shutdown signals the Exit channel, and closes it once. // Subsequent calls to this method do nothing func (ctrl *Control) Shutdown() { ctrl.shutdown.Do(func() { + ctrl.Lock() + defer ctrl.Unlock() close(ctrl.Exit) + ctrl.exited = true }) } +// HasExited checks to see if the control is still active +func (ctrl *Control) HasExited() bool { + ctrl.Lock() + defer ctrl.Unlock() + return ctrl.exited +} + // NewControl creates a new control group, initialized to the not ready state // (Ready WaitGroup semaphore to 1). Exit Go Channel is also created with a buffer of 1. func NewControl(wgCount int) *Control { diff --git a/vendor/vendor.json b/vendor/vendor.json index 26f253d..af1c174 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -66,10 +66,10 @@ "tree": true }, { - "checksumSHA1": "Tz3FMUl0EQFg0qe0IhTlyvGybTE=", + "checksumSHA1": "ArTICAlBYE6ni6VZWX6L2pzxa9M=", "path": "github.com/streadway/amqp", - "revision": "2e25825abdbd7752ff08b270d313b93519a0a232", - "revisionTime": "2016-03-11T21:55:03Z", + "revision": "d75c3a341ff43309ad0cb69ac8bdbd1d8772775f", + "revisionTime": "2017-02-03T13:40:38Z", "tree": true }, { @@ -79,6 +79,13 @@ "revisionTime": "2016-06-15T09:26:46Z", "tree": true }, + { + "checksumSHA1": "rG2lDtRjWtji8JUE6LXiiKzziQs=", + "path": "golang.org/x/crypto/acme", + "revision": "3cb07270c9455e8ad27956a70891c962d121a228", + "revisionTime": "2017-03-30T16:02:45Z", + "tree": true + }, { "checksumSHA1": "vE43s37+4CJ2CDU6TlOUOYE0K9c=", "path": "golang.org/x/crypto/bcrypt", @@ -92,6 +99,13 @@ "revision": "6ab629be5e31660579425a738ba8870beb5b7404", "revisionTime": "2016-09-19T18:57:51Z", "tree": true + }, + { + "checksumSHA1": "Y+HGqEkYM15ir+J93MEaHdyFy0c=", + "path": "golang.org/x/net/context", + "revision": "ffcf1bedda3b04ebb15a168a59800a73d6dc0f4d", + "revisionTime": "2017-03-29T01:43:45Z", + "tree": true } ], "rootPath": "github.com/CodeCollaborate/Server"