Merge branch 'bjorn/asn1/fix-huge-seqof/ERIERL-859/OTP-18245' into maint-24

* bjorn/asn1/fix-huge-seqof/ERIERL-859/OTP-18245:
  asn1: Support SEQUENCE OF with 16384 or more items

6 files changed, 232 insertions, 44 deletions

diff --git a/lib/asn1/src/asn1ct_constructed_per.erl b/lib/asn1/src/asn1ct_constructed_per.erl
index aff383479b..ec5894b458 100644
--- a/lib/asn1/src/asn1ct_constructed_per.erl
+++ b/lib/asn1/src/asn1ct_constructed_per.erl
@@ -730,23 +730,55 @@ gen_decode_sof(Erules, Typename, SeqOrSetOf, #type{}=D) ->
     do_gen_decode_sof(Erules, Typename, SeqOrSetOf, D),
     emit([".",nl,nl]).
 
-do_gen_decode_sof(Erules, Typename, SeqOrSetOf, D) ->
+do_gen_decode_sof(Erules, TypeName, SeqOrSetOf, D) ->
+    case asn1ct_imm:effective_constraint(bitstring, D#type.constraint) of
+        no ->
+            %% Could be fragmented.
+            do_gen_decode_fragmented(Erules, TypeName, SeqOrSetOf, D);
+        SizeConstraint ->
+            do_gen_decode_sof_plain(Erules, TypeName, SeqOrSetOf, D, SizeConstraint)
+    end.
+
+do_gen_decode_fragmented(Erules, TypeName, SeqOrSetOf, D) ->
     {_SeqOrSetOf,ComponentType} = D#type.def,
-    SizeConstraint = asn1ct_imm:effective_constraint(bitstring,
-						     D#type.constraint),
-    ObjFun =
-	case D#type.tablecinf of
-	    [{objfun,_}|_R] ->
-		", ObjFun";
-	    _ ->
-		""
-	end,
+    ObjFun = obj_fun_arg(D),
+    Key = erlang:md5(term_to_binary({fragmented,TypeName,SeqOrSetOf,ComponentType})),
+    Gen = fun(_Fd, Name) ->
+                  do_gen_dec_fragmented_1(Erules, Name, TypeName,
+                                          SeqOrSetOf, ComponentType, D)
+	  end,
+    F = asn1ct_func:call_gen("dec_components", Key, Gen),
+    emit([{asis,F}, "(Bytes", ObjFun,", [])"]).
+
+do_gen_dec_fragmented_1(Erules, Name, TypeName, SeqOrSetOf, ComponentType, D) ->
+    ObjFun = obj_fun_arg(D),
+    emit([{asis,Name}, "(Bytes", ObjFun, ", Acc) ->", nl]),
+    {Num,Buf} = gen_decode_length(no, Erules),
+    Key = erlang:md5(term_to_binary({TypeName,SeqOrSetOf,ComponentType})),
+    Gen = fun(_Fd, Name2) ->
+		  gen_decode_sof_components(Erules, Name2,
+					    TypeName, SeqOrSetOf,
+					    ComponentType, false)
+	  end,
+    F = asn1ct_func:call_gen("dec_fragment", Key, Gen),
+    emit([",",nl,
+          "{Acc1,Buf1} = ",
+	  {asis,F}, "(", Num, ", ", Buf, ObjFun, ", Acc),",nl]),
+    emit(["if ",Num," >= 16384 ->",nl,
+          {asis,Name},"(Buf1", ObjFun, ", Acc1);",nl,
+          "true ->",nl,
+          "{lists:reverse(Acc1),Buf1}",nl,
+          "end.",nl]).
+
+do_gen_decode_sof_plain(Erules, TypeName, SeqOrSetOf, D, SizeConstraint) ->
+    {_SeqOrSetOf,ComponentType} = D#type.def,
+    ObjFun = obj_fun_arg(D),
     {Num,Buf} = gen_decode_length(SizeConstraint, Erules),
-    Key = erlang:md5(term_to_binary({Typename,SeqOrSetOf,ComponentType})),
+    Key = erlang:md5(term_to_binary({TypeName,SeqOrSetOf,ComponentType})),
     Gen = fun(_Fd, Name) ->
 		  gen_decode_sof_components(Erules, Name,
-					    Typename, SeqOrSetOf,
-					    ComponentType)
+					    TypeName, SeqOrSetOf,
+					    ComponentType, true)
 	  end,
     F = asn1ct_func:call_gen("dec_components", Key, Gen),
     emit([",",nl,
@@ -759,16 +791,16 @@ gen_decode_length(Constraint, Erule) ->
     Imm = asn1ct_imm:per_dec_length(Constraint, true, is_aligned(Erule)),
     asn1ct_imm:dec_slim_cg(Imm, "Bytes").
 
-gen_decode_sof_components(Erule, Name, Typename, SeqOrSetOf, Cont) ->
-    {ObjFun,ObjFun_Var} =
-	case Cont#type.tablecinf of
-	    [{objfun,_}|_R] ->
-		{", ObjFun",", _"};
-	    _ ->
-		{"",""}
-	end,
-    emit([{asis,Name},"(0, Bytes",ObjFun_Var,", Acc) ->",nl,
-	  "{lists:reverse(Acc),Bytes};",nl]),
+gen_decode_sof_components(Erule, Name, Typename, SeqOrSetOf, Cont, Reverse) ->
+    ObjFun = obj_fun_arg(Cont),
+    ObjFunPat = obj_fun_pat(Cont),
+    emit([{asis,Name},"(0, Bytes",ObjFunPat,", Acc) ->",nl]),
+    case Reverse of
+        true ->
+            emit(["{lists:reverse(Acc),Bytes};",nl]);
+        false ->
+            emit(["{Acc,Bytes};",nl])
+    end,
     emit([{asis,Name},"(Num, Bytes",ObjFun,", Acc) ->",nl,
 	  "{Term,Remain} = "]),
     Constructed_Suffix = asn1ct_gen:constructed_suffix(SeqOrSetOf,
@@ -794,6 +826,15 @@ gen_decode_sof_components(Erule, Name, Typename, SeqOrSetOf, Cont) ->
     end,
     emit([{asis,Name},"(Num-1, Remain",ObjFun,", [Term|Acc]).",nl]).
 
+obj_fun_arg(#type{tablecinf=[{objfun,_}|_]}) ->
+    ", ObjFun";
+obj_fun_arg(#type{}) ->
+    "".
+
+obj_fun_pat(#type{tablecinf=[{objfun,_}|_]}) ->
+    ", _";
+obj_fun_pat(#type{}) ->
+    "".
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %% General and special help functions (not exported)
diff --git a/lib/asn1/src/asn1ct_imm.erl b/lib/asn1/src/asn1ct_imm.erl
index 231048694a..e70a91ed9e 100644
--- a/lib/asn1/src/asn1ct_imm.erl
+++ b/lib/asn1/src/asn1ct_imm.erl
@@ -381,22 +381,44 @@ per_enc_optional(Val, {call,M,F,A}) ->
       [[{eq,Tmp,true},Zero],['_',One]]}].
 
 per_enc_sof(Val0, Constraint, ElementVar, ElementImm, Aligned) ->
-    {B,[Val,Len]} = mk_vars(Val0, [len]),
-    SzConstraint = effective_constraint(bitstring, Constraint),
-    LenImm = enc_length(Len, SzConstraint, Aligned),
-    Lc0 = [{lc,ElementImm,{var,atom_to_list(ElementVar)},Val}],
-    Lc = opt_lc(Lc0, LenImm),
-    PreBlock = B ++ [{call,erlang,length,[Val],Len}],
-    case LenImm of
-	[{'cond',[[C|Action]]}] ->
-	    PreBlock ++ [{'cond',[[C|Action++Lc]]}];
-	[{sub,_,_,_}=Sub,{'cond',[[C|Action]]}] ->
-	    PreBlock ++
-		[Sub,{'cond',[[C|Action++Lc]]}];
-	EncLen ->
-	    PreBlock ++ EncLen ++ Lc
+    case effective_constraint(bitstring, Constraint) of
+        no ->
+            per_enc_sof_fragmented(Val0, ElementVar, ElementImm, Aligned);
+        SzConstraint ->
+            {B,[Val,Len]} = mk_vars(Val0, [len]),
+            LenImm = enc_sof_length(Len, SzConstraint, Aligned),
+            Lc0 = [{lc,ElementImm,{var,atom_to_list(ElementVar)},Val}],
+            Lc = opt_lc(Lc0, LenImm),
+            PreBlock = B ++ [{call,erlang,length,[Val],Len}],
+            case LenImm of
+                [{'cond',[[C|Action]]}] ->
+                    PreBlock ++ [{'cond',[[C|Action++Lc]]}];
+                [{sub,_,_,_}=Sub,{'cond',[[C|Action]]}] ->
+                    PreBlock ++
+                        [Sub,{'cond',[[C|Action++Lc]]}];
+                EncLen ->
+                    PreBlock ++ EncLen ++ Lc
+            end
     end.
 
+per_enc_sof_fragmented(Val0, ElementVar, ElementImm, Aligned) ->
+    {B,[Val,Len,Fun]} = mk_vars(Val0, [len,fn]),
+    Lc = [{lc,ElementImm,{var,atom_to_list(ElementVar)},Val}],
+    PreBlock = B ++ [{call,erlang,length,[Val],Len}],
+    U = unit(1, Aligned),
+    EncFragmented =
+        [{'fun',
+          [{var,atom_to_list(ElementVar)}],
+          ElementImm,
+          Fun},
+         {call,enc_mod(Aligned),encode_fragmented_sof,[Fun,Val,Len]}],
+    CondImm = build_cond([[{lt,Len,128},
+                           {put_bits,Len,8,U}|Lc],
+                          [{lt,Len,16384},
+                           {put_bits,2,2,U},{put_bits,Len,14,[1]}|Lc],
+                          ['_'|EncFragmented]]),
+    PreBlock ++ CondImm.
+
 enc_absent(Val0, {call,M,F,A}, Body) ->
     {B,[Var,Tmp]} = mk_vars(Val0, [tmp]),
     B++[{call,M,F,[Var|A],Tmp},
@@ -586,7 +608,10 @@ decode_unconstrained_length(AllowZero, Aligned) ->
 	      {value,{get_bits,7,[1|Zero]}}},
 	     {test,{get_bits,1,[1|Al]},1,
 	      {test,{get_bits,1,[1]},0,
-	       {value,{get_bits,14,[1|Zero]}}}}]}.
+	       {value,{get_bits,14,[1|Zero]}}}},
+	     {test,{get_bits,1,[1|Al]},1,
+	      {test,{get_bits,1,[1]},1,
+	       {value,{mul,{get_bits,6,[1|Zero]},16384}}}}]}.
 
 uper_num_bits(N) ->
     uper_num_bits(N, 1, 0).
@@ -751,6 +776,9 @@ opt_al({value,E0}, A0) ->
 opt_al({add,E0,I}, A0) when is_integer(I) ->
     {E,A} = opt_al(E0, A0),
     {{add,E,I},A};
+opt_al({mul,E0,I}, A0) when is_integer(I) ->
+    {E,A} = opt_al(E0, A0),
+    {{mul,E,I},A};
 opt_al({test,E0,V,B0}, A0) ->
     {E,A1} = opt_al(E0, A0),
     {B,A2} = opt_al(B0, A1),
@@ -838,6 +866,10 @@ flatten({add,E0,I}, Buf0, St0) ->
     {{Src,Buf},Pre,St1} = flatten(E0, Buf0, St0),
     {Dst,St} = new_var("Add", St1),
     {{Dst,Buf},Pre++[{add,Src,I,Dst}],St};
+flatten({mul,E0,I}, Buf0, St0) ->
+    {{Src,Buf},Pre,St1} = flatten(E0, Buf0, St0),
+    {Dst,St} = new_var("Mul", St1),
+    {{Dst,Buf},Pre++[{mul,Src,I,Dst}],St};
 flatten({'case',Cs0}, Buf0, St0) ->
     {Dst,St1} = new_var_pair(St0),
     {Cs1,St} = flatten_cs(Cs0, Buf0, St1),
@@ -951,6 +983,9 @@ dcg_list_outside([{'map',Val,Cs,Dst}|T]) ->
 dcg_list_outside([{add,S1,S2,Dst}|T]) ->
     emit([Dst," = ",S1," + ",S2]),
     iter_dcg_list_outside(T);
+dcg_list_outside([{mul,S1,S2,Dst}|T]) ->
+    emit([Dst," = ",S1," * ",S2]),
+    iter_dcg_list_outside(T);
 dcg_list_outside([{return,{V,Buf}}|T]) ->
     emit(["{",V,",",Buf,"}"]),
     iter_dcg_list_outside(T);
@@ -1055,6 +1090,7 @@ split_off_nonbuilding(Imm) ->
 is_nonbuilding({assign,_,_}) -> true;
 is_nonbuilding({call,_,_,_,_}) -> true;
 is_nonbuilding({comment,_}) -> true;
+is_nonbuilding({'fun',_,_,_}) -> true;
 is_nonbuilding({lc,_,_,_,_}) -> true;
 is_nonbuilding({set,_,_}) -> true;
 is_nonbuilding({list,_,_}) -> true;
@@ -1239,23 +1275,23 @@ per_enc_length(Bin, Unit0, Len, Sv, Aligned, Type) when is_integer(Sv) ->
     Pb = {put_bits,Bin,binary,U},
     [{'cond',[[{eq,Len,Sv},Pb]]}].
 
-enc_length(Len, no, Aligned) ->
+enc_sof_length(Len, no, Aligned) ->
     U = unit(1, Aligned),
     build_cond([[{lt,Len,128},
 		 {put_bits,Len,8,U}],
 		[{lt,Len,16384},
 		 {put_bits,2,2,U},{put_bits,Len,14,[1]}]]);
-enc_length(Len, {{Lb,Ub},[]}, Aligned) ->
+enc_sof_length(Len, {{Lb,Ub},[]}, Aligned) ->
     {Prefix,Check,PutLen} = per_enc_constrained(Len, Lb, Ub, Aligned),
     NoExt = {put_bits,0,1,[1]},
-    [{'cond',ExtConds0}] = enc_length(Len, no, Aligned),
+    [{'cond',ExtConds0}] = enc_sof_length(Len, no, Aligned),
     Ext = {put_bits,1,1,[1]},
     ExtConds = prepend_to_cond(ExtConds0, Ext),
     build_length_cond(Prefix, [[Check,NoExt|PutLen]|ExtConds]);
-enc_length(Len, {Lb,Ub}, Aligned) when is_integer(Lb) ->
+enc_sof_length(Len, {Lb,Ub}, Aligned) when is_integer(Lb) ->
     {Prefix,Check,PutLen} = per_enc_constrained(Len, Lb, Ub, Aligned),
     build_length_cond(Prefix, [[Check|PutLen]]);
-enc_length(Len, Sv, _Aligned) when is_integer(Sv) ->
+enc_sof_length(Len, Sv, _Aligned) when is_integer(Sv) ->
     [{'cond',[[{eq,Len,Sv}]]}].
 
 extensions_bitmap(Vs, Undefined) ->
@@ -1730,6 +1766,9 @@ enc_make_cons({integer,Int}, {cons,{binary,H},T}) ->
 enc_make_cons(H, T) ->
     {cons,H,T}.
 
+enc_pre_cg_nonbuilding({'fun',Args,B0,Dst}, StL) ->
+    B = enc_pre_cg_1(B0, StL, outside_seq),
+    {'fun',Args,B,Dst};
 enc_pre_cg_nonbuilding({lc,B0,Var,List,Dst}, StL) ->
     B = enc_pre_cg_1(B0, StL, outside_seq),
     {lc,B,Var,List,Dst};
@@ -1943,6 +1982,9 @@ enc_opt({cons,H0,T0}, St0) ->
     {{cons,H,T},St#ost{t=t_cons(TypeH, TypeT)}};
 enc_opt({error,_}=Imm, St) ->
     {Imm,St#ost{t=t_any()}};
+enc_opt({'fun',_,_,Dst}=Imm, St0) ->
+    St = set_type(Dst, t_any(), St0),
+    {Imm,St};
 enc_opt({integer,V}, St) ->
     {{integer,subst(V, St)},St#ost{t=t_integer()}};
 enc_opt({lc,E0,B,C}, St) ->
@@ -2365,6 +2407,13 @@ enc_cg({error,Error}) when is_function(Error, 0) ->
 enc_cg({error,{Tag,Var0}}) ->
     Var = mk_val(Var0),
     emit(["exit({error,{asn1,{",Tag,",",Var,"}}})"]);
+enc_cg({'fun',Args,Body,Dst0}) ->
+    Dst = mk_val(Dst0),
+    emit([Dst," = fun("]),
+    _ = [emit(mk_val(A)) || A <- Args],
+    emit(") -> "),
+    enc_cg(Body),
+    emit(" end");
 enc_cg({integer,Int}) ->
     emit(mk_val(Int));
 enc_cg({lc,Body,Var,List}) ->
@@ -2738,6 +2787,8 @@ per_fixup([{call_gen,_,_,_,_,_}=H|T]) ->
     [H|per_fixup(T)];
 per_fixup([{error,_}=H|T]) ->
     [H|per_fixup(T)];
+per_fixup([{'fun',Args,Body,Dst}|T]) ->
+    [{'fun',Args,per_fixup(Body),Dst}|per_fixup(T)];
 per_fixup([{lc,B,V,L}|T]) ->
     [{lc,per_fixup(B),V,L}|per_fixup(T)];
 per_fixup([{lc,B,V,L,Dst}|T]) ->
diff --git a/lib/asn1/src/asn1rtt_per.erl b/lib/asn1/src/asn1rtt_per.erl
index 753a38aa6e..70b1880693 100644
--- a/lib/asn1/src/asn1rtt_per.erl
+++ b/lib/asn1/src/asn1rtt_per.erl
@@ -19,7 +19,7 @@
 %%
 -module(asn1rtt_per).
 
--export([skipextensions/3,complete/1]).
+-export([skipextensions/3,complete/1,encode_fragmented_sof/3]).
 
 skipextensions(Bytes0, Nr, ExtensionBitstr) when is_bitstring(ExtensionBitstr) ->
     Prev = Nr - 1,
@@ -119,3 +119,31 @@ complete(Bin, Bits, More) when is_binary(Bin) ->
     [Bin|complete([], Bits, More)];
 complete(Bin, Bits, More) ->
     [Bin|complete([], Bits+bit_size(Bin), More)].
+
+-define('16K',16384).
+
+encode_fragmented_sof(Fun, Comps, Len) ->
+    encode_fragmented_sof_1(Fun, Comps, Len, 4).
+
+encode_fragmented_sof_1(Encoder, Comps0, Len0, N) ->
+    SegSz = N * ?'16K',
+    if
+        Len0 >= SegSz ->
+            {Comps,B} = encode_components(Comps0, Encoder, SegSz, []),
+            Len = Len0 - SegSz,
+            [align,<<3:2,N:6>>,B|encode_fragmented_sof_1(Encoder, Comps, Len, N)];
+        N > 1 ->
+            encode_fragmented_sof_1(Encoder, Comps0, Len0, N - 1);
+        Len0 < 128 ->
+            {[],B} = encode_components(Comps0, Encoder, Len0, []),
+            [align,Len0|B];
+        Len0 < ?'16K' ->
+            {[],B} = encode_components(Comps0, Encoder, Len0, []),
+            [align,<<2:2,Len0:14>>|B]
+    end.
+
+encode_components(Cs, _Encoder, 0, Acc) ->
+    {Cs,lists:reverse(Acc)};
+encode_components([C|Cs], Encoder, Size, Acc) ->
+    B = Encoder(C),
+    encode_components(Cs, Encoder, Size - 1, [B|Acc]).
diff --git a/lib/asn1/src/asn1rtt_uper.erl b/lib/asn1/src/asn1rtt_uper.erl
index 0ab8fab141..f51cb979b7 100644
--- a/lib/asn1/src/asn1rtt_uper.erl
+++ b/lib/asn1/src/asn1rtt_uper.erl
@@ -22,6 +22,7 @@
 
 -export([skipextensions/3]).
 -export([complete/1, complete_NFP/1]).
+-export([encode_fragmented_sof/3]).
 
 skipextensions(Bytes0, Nr, ExtensionBitstr) when is_bitstring(ExtensionBitstr) ->
     Prev = Nr - 1,
@@ -75,3 +76,31 @@ complete_NFP(InList) when is_list(InList) ->
     list_to_bitstring(InList);
 complete_NFP(InList) when is_bitstring(InList) ->
     InList.
+
+-define('16K',16384).
+
+encode_fragmented_sof(Fun, Comps, Len) ->
+    encode_fragmented_sof_1(Fun, Comps, Len, 4).
+
+encode_fragmented_sof_1(Encoder, Comps0, Len0, N) ->
+    SegSz = N * ?'16K',
+    if
+        Len0 >= SegSz ->
+            {Comps,B} = encode_components(Comps0, Encoder, SegSz, []),
+            Len = Len0 - SegSz,
+            [<<3:2,N:6>>,B|encode_fragmented_sof_1(Encoder, Comps, Len, N)];
+        N > 1 ->
+            encode_fragmented_sof_1(Encoder, Comps0, Len0, N - 1);
+        Len0 < 128 ->
+            {[],B} = encode_components(Comps0, Encoder, Len0, []),
+            [Len0|B];
+        Len0 < ?'16K' ->
+            {[],B} = encode_components(Comps0, Encoder, Len0, []),
+            [<<2:2,Len0:14>>|B]
+    end.
+
+encode_components(Cs, _Encoder, 0, Acc) ->
+    {Cs,lists:reverse(Acc)};
+encode_components([C|Cs], Encoder, Size, Acc) ->
+    B = Encoder(C),
+    encode_components(Cs, Encoder, Size - 1, [B|Acc]).
diff --git a/lib/asn1/test/asn1_SUITE_data/Fragmented.asn1 b/lib/asn1/test/asn1_SUITE_data/Fragmented.asn1
index bfc939737f..e784feff8c 100644
--- a/lib/asn1/test/asn1_SUITE_data/Fragmented.asn1
+++ b/lib/asn1/test/asn1_SUITE_data/Fragmented.asn1
@@ -21,4 +21,13 @@ PDU ::= SEQUENCE {
   arg FUNCTION.&ArgumentType ({ObjSet}{@code})
 }
 
+IntBoolSeqs ::= SEQUENCE (SIZE (1..65536)) OF IntBoolSeq
+
+IntBoolSeqsU ::= SEQUENCE OF IntBoolSeq
+
+IntBoolSeq ::= SEQUENCE {
+  a INTEGER,
+  b BOOLEAN
+}
+
 END
diff --git a/lib/asn1/test/testFragmented.erl b/lib/asn1/test/testFragmented.erl
index bd63bd83fc..fe23b13d3a 100644
--- a/lib/asn1/test/testFragmented.erl
+++ b/lib/asn1/test/testFragmented.erl
@@ -36,7 +36,37 @@ main(_Erule) ->
 				     K8,K8,K8,K8,K8,K8]}),
     roundtrip('PDU', {'PDU',1,false,[K8,K8,K8,K8,K8,K8,K8,K8,
 				     K8,K8,K8,K8,K8,K8,K8,K8]}),
+
+    K16 = 16384,
+    K64 = 4 * K16,
+    K144 = 2 * K64 + K16,
+    roundtrips([1, 2, 3, 17,
+                K16-1, K16, K16+1,
+                2*K16-1, 2*K16, 2*K16+1,
+                3*K16-1, 3*K16, 3*K16+1,
+                K64-1, K64, K64+1,
+                K64+K16,
+                K144-1, K144, K144+1]),
+    ok.
+
+roundtrips([Size|Sizes]) ->
+    L = make_seq(Size, []),
+    io:format("~p: ~P\n", [Size,L,6]),
+    roundtrip('IntBoolSeqsU', L),
+    if
+        Size =< 65536 ->
+            roundtrip('IntBoolSeqs', L);
+        true ->
+            ok
+    end,
+    roundtrips(Sizes);
+roundtrips([]) ->
     ok.
 
 roundtrip(T, V) ->
     asn1_test_lib:roundtrip('Fragmented', T, V).
+
+make_seq(0, Acc) ->
+    Acc;
+make_seq(N, Acc) ->
+    make_seq(N - 1, [{'IntBoolSeq',N,N rem 7 =:= 0}|Acc]).