Fix AffineNormalizer returning a proper subgroup

Changelog

@@ -1,3 +1,10 @@
+Unreleased
+
+   - fixed AffineNormalizer, which returned a proper subgroup of the affine
+     normalizer for some space groups (e.g. Pc), because the stabilizer
+     computation used incomplete Schreier generators
+     (thanks to lan496 for reporting).
+
 Version 4.1.31
 
    - fixed a bug in RepresentativeAction for affine cryst groups

gap/cryst2.gi

@@ -774,7 +774,8 @@ InstallMethod( AffineNormalizer, "for SpaceGroup acting OnRight", true,
 function( S )
 
     local d, P, H, T, N, Pgens, Sgens, invT, gens, Pi, Si, hom, opr, orb, 
-          g, m, set, rep, lst, pnt, img, t, sch, n, nn, normgens, TN, AN;
+          g, m, set, ind, rep, lst, i, j, k, pnt, img, t, sch, n, nn,
+          normgens, TN, AN;
 
     d := DimensionOfMatrixGroup( S ) - 1;
     P := PointGroup( S );
@@ -819,26 +820,44 @@ function( S )
         Add( orb, [ m, m[d+1]{[1..d]} ] );
     od;
     orb := [ orb ];
-    set := ShallowCopy( orb );
+
+    set := [ orb[1] ];   # the points of orb, sorted, for binary search
+    ind := [ 1 ];        # ind[k] is the position of set[k] in orb
 
     rep := [ One( N ) ];
     lst := [];
-    for pnt  in orb  do
+    i := 0;
+    while i < Length( orb ) do
+        i := i+1;
+        pnt := orb[i];
         for g  in gens  do
             img := List( pnt, x -> opr( x, g ) );
-            if not img in set  then
-                Add( orb, img );
-                AddSet( set, img );
-                Add( rep, rep[Position(orb,pnt)]*g );
+            k := PositionSorted( set, img );
+            if k <= Length( set ) and set[k] = img then
+                j := ind[k];
             else
-                t := AffineLift( img, d );
-                if t<>[] then
-                    sch := rep[Position(orb,pnt)]*g;
-                    n := IdentityMat( d+1 );
-                    n{[1..d]}{[1..d]} := sch;
-                    n[d+1]{[1..d]} := t;
-                    AddSet( lst, n );
-                fi;
+                Add( orb, img );
+                Add( rep, rep[i]*g );
+                j := Length( orb );
+                Add( set, img, k );
+                Add( ind, j, k );
+            fi;
+            # rep[i]*g maps the base point to img; if img lifts, it
+            # normalizes S once combined with the translation t
+            t := AffineLift( img, d );
+            if t<>[] then
+                n := IdentityMat( d+1 );
+                n{[1..d]}{[1..d]} := rep[i]*g;
+                n[d+1]{[1..d]} := t;
+                AddSet( lst, n );
+            fi;
+            # Schreier generator for the stabilizer of the base point;
+            # it fixes all translations, so it lifts with t = 0
+            sch := rep[i]*g*rep[j]^-1;
+            if sch <> One( N ) then
+                n := IdentityMat( d+1 );
+                n{[1..d]}{[1..d]} := sch;
+                AddSet( lst, n );
             fi;
         od;
     od;

tst/cryst.tst

@@ -305,4 +305,16 @@ gap> L := AsList( C[5] );
 gap> List(L, x -> RepresentativeAction( G, L[1], x, OnPoints ) );;
 gap> List( C, x -> Normalizer( G, Representative(x) ) );;
 
+# https://github.com/gap-packages/cryst/issues/60: the affine normalizer of
+# Pc must contain elements whose linear part is not block triangular
+gap> S := SpaceGroupOnRightIT( 3, 7 );;
+gap> if IsPackageMarkedForLoading( "CaratInterface", "" ) then
+>   gen := GeneratorsOfGroup( AffineNormalizer( S ) );;
+>   Print( ForAll( gen, x -> ForAll( GeneratorsOfGroup( S ), g -> g^x in S ) ),
+>          " ", ForAny( gen, x -> x[3][1] <> 0 ), "\n" );
+> else
+>   Print( "true true\n" );
+> fi;
+true true
+
 gap> STOP_TEST( "cryst.tst", 10000 );