+53 -600

api/tangled/cbor_gen.go

··· 2728 2728 2729 2729 return nil 2730 2730 } 2731 2731 - func (t *Pipeline_Dependency) MarshalCBOR(w io.Writer) error { 2732 2732 - if t == nil { 2733 2733 - _, err := w.Write(cbg.CborNull) 2734 2734 - return err 2735 2735 - } 2736 2736 - 2737 2737 - cw := cbg.NewCborWriter(w) 2738 2738 - 2739 2739 - if _, err := cw.Write([]byte{162}); err != nil { 2740 2740 - return err 2741 2741 - } 2742 2742 - 2743 2743 - // t.Packages ([]string) (slice) 2744 2744 - if len("packages") > 1000000 { 2745 2745 - return xerrors.Errorf("Value in field \"packages\" was too long") 2746 2746 - } 2747 2747 - 2748 2748 - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("packages"))); err != nil { 2749 2749 - return err 2750 2750 - } 2751 2751 - if _, err := cw.WriteString(string("packages")); err != nil { 2752 2752 - return err 2753 2753 - } 2754 2754 - 2755 2755 - if len(t.Packages) > 8192 { 2756 2756 - return xerrors.Errorf("Slice value in field t.Packages was too long") 2757 2757 - } 2758 2758 - 2759 2759 - if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Packages))); err != nil { 2760 2760 - return err 2761 2761 - } 2762 2762 - for _, v := range t.Packages { 2763 2763 - if len(v) > 1000000 { 2764 2764 - return xerrors.Errorf("Value in field v was too long") 2765 2765 - } 2766 2766 - 2767 2767 - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(v))); err != nil { 2768 2768 - return err 2769 2769 - } 2770 2770 - if _, err := cw.WriteString(string(v)); err != nil { 2771 2771 - return err 2772 2772 - } 2773 2773 - 2774 2774 - } 2775 2775 - 2776 2776 - // t.Registry (string) (string) 2777 2777 - if len("registry") > 1000000 { 2778 2778 - return xerrors.Errorf("Value in field \"registry\" was too long") 2779 2779 - } 2780 2780 - 2781 2781 - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("registry"))); err != nil { 2782 2782 - return err 2783 2783 - } 2784 2784 - if _, err := cw.WriteString(string("registry")); err != nil { 2785 2785 - return err 2786 2786 - } 2787 2787 - 2788 2788 - if len(t.Registry) > 1000000 { 2789 2789 - return xerrors.Errorf("Value in field t.Registry was too long") 2790 2790 - } 2791 2791 - 2792 2792 - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Registry))); err != nil { 2793 2793 - return err 2794 2794 - } 2795 2795 - if _, err := cw.WriteString(string(t.Registry)); err != nil { 2796 2796 - return err 2797 2797 - } 2798 2798 - return nil 2799 2799 - } 2800 2800 - 2801 2801 - func (t *Pipeline_Dependency) UnmarshalCBOR(r io.Reader) (err error) { 2802 2802 - *t = Pipeline_Dependency{} 2803 2803 - 2804 2804 - cr := cbg.NewCborReader(r) 2805 2805 - 2806 2806 - maj, extra, err := cr.ReadHeader() 2807 2807 - if err != nil { 2808 2808 - return err 2809 2809 - } 2810 2810 - defer func() { 2811 2811 - if err == io.EOF { 2812 2812 - err = io.ErrUnexpectedEOF 2813 2813 - } 2814 2814 - }() 2815 2815 - 2816 2816 - if maj != cbg.MajMap { 2817 2817 - return fmt.Errorf("cbor input should be of type map") 2818 2818 - } 2819 2819 - 2820 2820 - if extra > cbg.MaxLength { 2821 2821 - return fmt.Errorf("Pipeline_Dependency: map struct too large (%d)", extra) 2822 2822 - } 2823 2823 - 2824 2824 - n := extra 2825 2825 - 2826 2826 - nameBuf := make([]byte, 8) 2827 2827 - for i := uint64(0); i < n; i++ { 2828 2828 - nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) 2829 2829 - if err != nil { 2830 2830 - return err 2831 2831 - } 2832 2832 - 2833 2833 - if !ok { 2834 2834 - // Field doesn't exist on this type, so ignore it 2835 2835 - if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { 2836 2836 - return err 2837 2837 - } 2838 2838 - continue 2839 2839 - } 2840 2840 - 2841 2841 - switch string(nameBuf[:nameLen]) { 2842 2842 - // t.Packages ([]string) (slice) 2843 2843 - case "packages": 2844 2844 - 2845 2845 - maj, extra, err = cr.ReadHeader() 2846 2846 - if err != nil { 2847 2847 - return err 2848 2848 - } 2849 2849 - 2850 2850 - if extra > 8192 { 2851 2851 - return fmt.Errorf("t.Packages: array too large (%d)", extra) 2852 2852 - } 2853 2853 - 2854 2854 - if maj != cbg.MajArray { 2855 2855 - return fmt.Errorf("expected cbor array") 2856 2856 - } 2857 2857 - 2858 2858 - if extra > 0 { 2859 2859 - t.Packages = make([]string, extra) 2860 2860 - } 2861 2861 - 2862 2862 - for i := 0; i < int(extra); i++ { 2863 2863 - { 2864 2864 - var maj byte 2865 2865 - var extra uint64 2866 2866 - var err error 2867 2867 - _ = maj 2868 2868 - _ = extra 2869 2869 - _ = err 2870 2870 - 2871 2871 - { 2872 2872 - sval, err := cbg.ReadStringWithMax(cr, 1000000) 2873 2873 - if err != nil { 2874 2874 - return err 2875 2875 - } 2876 2876 - 2877 2877 - t.Packages[i] = string(sval) 2878 2878 - } 2879 2879 - 2880 2880 - } 2881 2881 - } 2882 2882 - // t.Registry (string) (string) 2883 2883 - case "registry": 2884 2884 - 2885 2885 - { 2886 2886 - sval, err := cbg.ReadStringWithMax(cr, 1000000) 2887 2887 - if err != nil { 2888 2888 - return err 2889 2889 - } 2890 2890 - 2891 2891 - t.Registry = string(sval) 2892 2892 - } 2893 2893 - 2894 2894 - default: 2895 2895 - // Field doesn't exist on this type, so ignore it 2896 2896 - if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { 2897 2897 - return err 2898 2898 - } 2899 2899 - } 2900 2900 - } 2901 2901 - 2902 2902 - return nil 2903 2903 - } 2904 2731 func (t *Pipeline_ManualTriggerData) MarshalCBOR(w io.Writer) error { 2905 2732 if t == nil { 2906 2733 _, err := w.Write(cbg.CborNull) ··· 3916 3743 3917 3744 return nil 3918 3745 } 3919 3919 - func (t *Pipeline_Step) MarshalCBOR(w io.Writer) error { 3920 3920 - if t == nil { 3921 3921 - _, err := w.Write(cbg.CborNull) 3922 3922 - return err 3923 3923 - } 3924 3924 - 3925 3925 - cw := cbg.NewCborWriter(w) 3926 3926 - fieldCount := 3 3927 3927 - 3928 3928 - if t.Environment == nil { 3929 3929 - fieldCount-- 3930 3930 - } 3931 3931 - 3932 3932 - if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { 3933 3933 - return err 3934 3934 - } 3935 3935 - 3936 3936 - // t.Name (string) (string) 3937 3937 - if len("name") > 1000000 { 3938 3938 - return xerrors.Errorf("Value in field \"name\" was too long") 3939 3939 - } 3940 3940 - 3941 3941 - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("name"))); err != nil { 3942 3942 - return err 3943 3943 - } 3944 3944 - if _, err := cw.WriteString(string("name")); err != nil { 3945 3945 - return err 3946 3946 - } 3947 3947 - 3948 3948 - if len(t.Name) > 1000000 { 3949 3949 - return xerrors.Errorf("Value in field t.Name was too long") 3950 3950 - } 3951 3951 - 3952 3952 - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Name))); err != nil { 3953 3953 - return err 3954 3954 - } 3955 3955 - if _, err := cw.WriteString(string(t.Name)); err != nil { 3956 3956 - return err 3957 3957 - } 3958 3958 - 3959 3959 - // t.Command (string) (string) 3960 3960 - if len("command") > 1000000 { 3961 3961 - return xerrors.Errorf("Value in field \"command\" was too long") 3962 3962 - } 3963 3963 - 3964 3964 - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("command"))); err != nil { 3965 3965 - return err 3966 3966 - } 3967 3967 - if _, err := cw.WriteString(string("command")); err != nil { 3968 3968 - return err 3969 3969 - } 3970 3970 - 3971 3971 - if len(t.Command) > 1000000 { 3972 3972 - return xerrors.Errorf("Value in field t.Command was too long") 3973 3973 - } 3974 3974 - 3975 3975 - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Command))); err != nil { 3976 3976 - return err 3977 3977 - } 3978 3978 - if _, err := cw.WriteString(string(t.Command)); err != nil { 3979 3979 - return err 3980 3980 - } 3981 3981 - 3982 3982 - // t.Environment ([]*tangled.Pipeline_Pair) (slice) 3983 3983 - if t.Environment != nil { 3984 3984 - 3985 3985 - if len("environment") > 1000000 { 3986 3986 - return xerrors.Errorf("Value in field \"environment\" was too long") 3987 3987 - } 3988 3988 - 3989 3989 - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("environment"))); err != nil { 3990 3990 - return err 3991 3991 - } 3992 3992 - if _, err := cw.WriteString(string("environment")); err != nil { 3993 3993 - return err 3994 3994 - } 3995 3995 - 3996 3996 - if len(t.Environment) > 8192 { 3997 3997 - return xerrors.Errorf("Slice value in field t.Environment was too long") 3998 3998 - } 3999 3999 - 4000 4000 - if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Environment))); err != nil { 4001 4001 - return err 4002 4002 - } 4003 4003 - for _, v := range t.Environment { 4004 4004 - if err := v.MarshalCBOR(cw); err != nil { 4005 4005 - return err 4006 4006 - } 4007 4007 - 4008 4008 - } 4009 4009 - } 4010 4010 - return nil 4011 4011 - } 4012 4012 - 4013 4013 - func (t *Pipeline_Step) UnmarshalCBOR(r io.Reader) (err error) { 4014 4014 - *t = Pipeline_Step{} 4015 4015 - 4016 4016 - cr := cbg.NewCborReader(r) 4017 4017 - 4018 4018 - maj, extra, err := cr.ReadHeader() 4019 4019 - if err != nil { 4020 4020 - return err 4021 4021 - } 4022 4022 - defer func() { 4023 4023 - if err == io.EOF { 4024 4024 - err = io.ErrUnexpectedEOF 4025 4025 - } 4026 4026 - }() 4027 4027 - 4028 4028 - if maj != cbg.MajMap { 4029 4029 - return fmt.Errorf("cbor input should be of type map") 4030 4030 - } 4031 4031 - 4032 4032 - if extra > cbg.MaxLength { 4033 4033 - return fmt.Errorf("Pipeline_Step: map struct too large (%d)", extra) 4034 4034 - } 4035 4035 - 4036 4036 - n := extra 4037 4037 - 4038 4038 - nameBuf := make([]byte, 11) 4039 4039 - for i := uint64(0); i < n; i++ { 4040 4040 - nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) 4041 4041 - if err != nil { 4042 4042 - return err 4043 4043 - } 4044 4044 - 4045 4045 - if !ok { 4046 4046 - // Field doesn't exist on this type, so ignore it 4047 4047 - if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { 4048 4048 - return err 4049 4049 - } 4050 4050 - continue 4051 4051 - } 4052 4052 - 4053 4053 - switch string(nameBuf[:nameLen]) { 4054 4054 - // t.Name (string) (string) 4055 4055 - case "name": 4056 4056 - 4057 4057 - { 4058 4058 - sval, err := cbg.ReadStringWithMax(cr, 1000000) 4059 4059 - if err != nil { 4060 4060 - return err 4061 4061 - } 4062 4062 - 4063 4063 - t.Name = string(sval) 4064 4064 - } 4065 4065 - // t.Command (string) (string) 4066 4066 - case "command": 4067 4067 - 4068 4068 - { 4069 4069 - sval, err := cbg.ReadStringWithMax(cr, 1000000) 4070 4070 - if err != nil { 4071 4071 - return err 4072 4072 - } 4073 4073 - 4074 4074 - t.Command = string(sval) 4075 4075 - } 4076 4076 - // t.Environment ([]*tangled.Pipeline_Pair) (slice) 4077 4077 - case "environment": 4078 4078 - 4079 4079 - maj, extra, err = cr.ReadHeader() 4080 4080 - if err != nil { 4081 4081 - return err 4082 4082 - } 4083 4083 - 4084 4084 - if extra > 8192 { 4085 4085 - return fmt.Errorf("t.Environment: array too large (%d)", extra) 4086 4086 - } 4087 4087 - 4088 4088 - if maj != cbg.MajArray { 4089 4089 - return fmt.Errorf("expected cbor array") 4090 4090 - } 4091 4091 - 4092 4092 - if extra > 0 { 4093 4093 - t.Environment = make([]*Pipeline_Pair, extra) 4094 4094 - } 4095 4095 - 4096 4096 - for i := 0; i < int(extra); i++ { 4097 4097 - { 4098 4098 - var maj byte 4099 4099 - var extra uint64 4100 4100 - var err error 4101 4101 - _ = maj 4102 4102 - _ = extra 4103 4103 - _ = err 4104 4104 - 4105 4105 - { 4106 4106 - 4107 4107 - b, err := cr.ReadByte() 4108 4108 - if err != nil { 4109 4109 - return err 4110 4110 - } 4111 4111 - if b != cbg.CborNull[0] { 4112 4112 - if err := cr.UnreadByte(); err != nil { 4113 4113 - return err 4114 4114 - } 4115 4115 - t.Environment[i] = new(Pipeline_Pair) 4116 4116 - if err := t.Environment[i].UnmarshalCBOR(cr); err != nil { 4117 4117 - return xerrors.Errorf("unmarshaling t.Environment[i] pointer: %w", err) 4118 4118 - } 4119 4119 - } 4120 4120 - 4121 4121 - } 4122 4122 - 4123 4123 - } 4124 4124 - } 4125 4125 - 4126 4126 - default: 4127 4127 - // Field doesn't exist on this type, so ignore it 4128 4128 - if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { 4129 4129 - return err 4130 4130 - } 4131 4131 - } 4132 4132 - } 4133 4133 - 4134 4134 - return nil 4135 4135 - } 4136 3746 func (t *Pipeline_TriggerMetadata) MarshalCBOR(w io.Writer) error { 4137 3747 if t == nil { 4138 3748 _, err := w.Write(cbg.CborNull) ··· 4609 4219 4610 4220 cw := cbg.NewCborWriter(w) 4611 4221 4612 4612 - if _, err := cw.Write([]byte{165}); err != nil { 4222 4222 + if _, err := cw.Write([]byte{164}); err != nil { 4223 4223 + return err 4224 4224 + } 4225 4225 + 4226 4226 + // t.Raw (string) (string) 4227 4227 + if len("raw") > 1000000 { 4228 4228 + return xerrors.Errorf("Value in field \"raw\" was too long") 4229 4229 + } 4230 4230 + 4231 4231 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("raw"))); err != nil { 4232 4232 + return err 4233 4233 + } 4234 4234 + if _, err := cw.WriteString(string("raw")); err != nil { 4235 4235 + return err 4236 4236 + } 4237 4237 + 4238 4238 + if len(t.Raw) > 1000000 { 4239 4239 + return xerrors.Errorf("Value in field t.Raw was too long") 4240 4240 + } 4241 4241 + 4242 4242 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Raw))); err != nil { 4243 4243 + return err 4244 4244 + } 4245 4245 + if _, err := cw.WriteString(string(t.Raw)); err != nil { 4613 4246 return err 4614 4247 } 4615 4248 ··· 4652 4285 return err 4653 4286 } 4654 4287 4655 4655 - // t.Steps ([]*tangled.Pipeline_Step) (slice) 4656 4656 - if len("steps") > 1000000 { 4657 4657 - return xerrors.Errorf("Value in field \"steps\" was too long") 4288 4288 + // t.Engine (string) (string) 4289 4289 + if len("engine") > 1000000 { 4290 4290 + return xerrors.Errorf("Value in field \"engine\" was too long") 4658 4291 } 4659 4292 4660 4660 - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("steps"))); err != nil { 4661 4661 - return err 4662 4662 - } 4663 4663 - if _, err := cw.WriteString(string("steps")); err != nil { 4293 4293 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("engine"))); err != nil { 4664 4294 return err 4665 4295 } 4666 4666 - 4667 4667 - if len(t.Steps) > 8192 { 4668 4668 - return xerrors.Errorf("Slice value in field t.Steps was too long") 4669 4669 - } 4670 4670 - 4671 4671 - if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Steps))); err != nil { 4672 4672 - return err 4673 4673 - } 4674 4674 - for _, v := range t.Steps { 4675 4675 - if err := v.MarshalCBOR(cw); err != nil { 4676 4676 - return err 4677 4677 - } 4678 4678 - 4679 4679 - } 4680 4680 - 4681 4681 - // t.Environment ([]*tangled.Pipeline_Pair) (slice) 4682 4682 - if len("environment") > 1000000 { 4683 4683 - return xerrors.Errorf("Value in field \"environment\" was too long") 4684 4684 - } 4685 4685 - 4686 4686 - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("environment"))); err != nil { 4687 4687 - return err 4688 4688 - } 4689 4689 - if _, err := cw.WriteString(string("environment")); err != nil { 4690 4690 - return err 4691 4691 - } 4692 4692 - 4693 4693 - if len(t.Environment) > 8192 { 4694 4694 - return xerrors.Errorf("Slice value in field t.Environment was too long") 4695 4695 - } 4696 4696 - 4697 4697 - if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Environment))); err != nil { 4296 4296 + if _, err := cw.WriteString(string("engine")); err != nil { 4698 4297 return err 4699 4298 } 4700 4700 - for _, v := range t.Environment { 4701 4701 - if err := v.MarshalCBOR(cw); err != nil { 4702 4702 - return err 4703 4703 - } 4704 4299 4300 4300 + if len(t.Engine) > 1000000 { 4301 4301 + return xerrors.Errorf("Value in field t.Engine was too long") 4705 4302 } 4706 4303 4707 4707 - // t.Dependencies ([]*tangled.Pipeline_Dependency) (slice) 4708 4708 - if len("dependencies") > 1000000 { 4709 4709 - return xerrors.Errorf("Value in field \"dependencies\" was too long") 4710 4710 - } 4711 4711 - 4712 4712 - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("dependencies"))); err != nil { 4304 4304 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Engine))); err != nil { 4713 4305 return err 4714 4306 } 4715 4715 - if _, err := cw.WriteString(string("dependencies")); err != nil { 4307 4307 + if _, err := cw.WriteString(string(t.Engine)); err != nil { 4716 4308 return err 4717 4717 - } 4718 4718 - 4719 4719 - if len(t.Dependencies) > 8192 { 4720 4720 - return xerrors.Errorf("Slice value in field t.Dependencies was too long") 4721 4721 - } 4722 4722 - 4723 4723 - if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Dependencies))); err != nil { 4724 4724 - return err 4725 4725 - } 4726 4726 - for _, v := range t.Dependencies { 4727 4727 - if err := v.MarshalCBOR(cw); err != nil { 4728 4728 - return err 4729 4729 - } 4730 4730 - 4731 4309 } 4732 4310 return nil 4733 4311 } ··· 4757 4335 4758 4336 n := extra 4759 4337 4760 4760 - nameBuf := make([]byte, 12) 4338 4338 + nameBuf := make([]byte, 6) 4761 4339 for i := uint64(0); i < n; i++ { 4762 4340 nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) 4763 4341 if err != nil { ··· 4773 4351 } 4774 4352 4775 4353 switch string(nameBuf[:nameLen]) { 4776 4776 - // t.Name (string) (string) 4354 4354 + // t.Raw (string) (string) 4355 4355 + case "raw": 4356 4356 + 4357 4357 + { 4358 4358 + sval, err := cbg.ReadStringWithMax(cr, 1000000) 4359 4359 + if err != nil { 4360 4360 + return err 4361 4361 + } 4362 4362 + 4363 4363 + t.Raw = string(sval) 4364 4364 + } 4365 4365 + // t.Name (string) (string) 4777 4366 case "name": 4778 4367 4779 4368 { ··· 4804 4393 } 4805 4394 4806 4395 } 4807 4807 - // t.Steps ([]*tangled.Pipeline_Step) (slice) 4808 4808 - case "steps": 4396 4396 + // t.Engine (string) (string) 4397 4397 + case "engine": 4809 4398 4810 4810 - maj, extra, err = cr.ReadHeader() 4811 4811 - if err != nil { 4812 4812 - return err 4813 4813 - } 4814 4814 - 4815 4815 - if extra > 8192 { 4816 4816 - return fmt.Errorf("t.Steps: array too large (%d)", extra) 4817 4817 - } 4818 4818 - 4819 4819 - if maj != cbg.MajArray { 4820 4820 - return fmt.Errorf("expected cbor array") 4821 4821 - } 4822 4822 - 4823 4823 - if extra > 0 { 4824 4824 - t.Steps = make([]*Pipeline_Step, extra) 4825 4825 - } 4826 4826 - 4827 4827 - for i := 0; i < int(extra); i++ { 4828 4828 - { 4829 4829 - var maj byte 4830 4830 - var extra uint64 4831 4831 - var err error 4832 4832 - _ = maj 4833 4833 - _ = extra 4834 4834 - _ = err 4835 4835 - 4836 4836 - { 4837 4837 - 4838 4838 - b, err := cr.ReadByte() 4839 4839 - if err != nil { 4840 4840 - return err 4841 4841 - } 4842 4842 - if b != cbg.CborNull[0] { 4843 4843 - if err := cr.UnreadByte(); err != nil { 4844 4844 - return err 4845 4845 - } 4846 4846 - t.Steps[i] = new(Pipeline_Step) 4847 4847 - if err := t.Steps[i].UnmarshalCBOR(cr); err != nil { 4848 4848 - return xerrors.Errorf("unmarshaling t.Steps[i] pointer: %w", err) 4849 4849 - } 4850 4850 - } 4851 4851 - 4852 4852 - } 4853 4853 - 4399 4399 + { 4400 4400 + sval, err := cbg.ReadStringWithMax(cr, 1000000) 4401 4401 + if err != nil { 4402 4402 + return err 4854 4403 } 4855 4855 - } 4856 4856 - // t.Environment ([]*tangled.Pipeline_Pair) (slice) 4857 4857 - case "environment": 4858 4404 4859 4859 - maj, extra, err = cr.ReadHeader() 4860 4860 - if err != nil { 4861 4861 - return err 4862 4862 - } 4863 4863 - 4864 4864 - if extra > 8192 { 4865 4865 - return fmt.Errorf("t.Environment: array too large (%d)", extra) 4866 4866 - } 4867 4867 - 4868 4868 - if maj != cbg.MajArray { 4869 4869 - return fmt.Errorf("expected cbor array") 4870 4870 - } 4871 4871 - 4872 4872 - if extra > 0 { 4873 4873 - t.Environment = make([]*Pipeline_Pair, extra) 4874 4874 - } 4875 4875 - 4876 4876 - for i := 0; i < int(extra); i++ { 4877 4877 - { 4878 4878 - var maj byte 4879 4879 - var extra uint64 4880 4880 - var err error 4881 4881 - _ = maj 4882 4882 - _ = extra 4883 4883 - _ = err 4884 4884 - 4885 4885 - { 4886 4886 - 4887 4887 - b, err := cr.ReadByte() 4888 4888 - if err != nil { 4889 4889 - return err 4890 4890 - } 4891 4891 - if b != cbg.CborNull[0] { 4892 4892 - if err := cr.UnreadByte(); err != nil { 4893 4893 - return err 4894 4894 - } 4895 4895 - t.Environment[i] = new(Pipeline_Pair) 4896 4896 - if err := t.Environment[i].UnmarshalCBOR(cr); err != nil { 4897 4897 - return xerrors.Errorf("unmarshaling t.Environment[i] pointer: %w", err) 4898 4898 - } 4899 4899 - } 4900 4900 - 4901 4901 - } 4902 4902 - 4903 4903 - } 4904 4904 - } 4905 4905 - // t.Dependencies ([]*tangled.Pipeline_Dependency) (slice) 4906 4906 - case "dependencies": 4907 4907 - 4908 4908 - maj, extra, err = cr.ReadHeader() 4909 4909 - if err != nil { 4910 4910 - return err 4911 4911 - } 4912 4912 - 4913 4913 - if extra > 8192 { 4914 4914 - return fmt.Errorf("t.Dependencies: array too large (%d)", extra) 4915 4915 - } 4916 4916 - 4917 4917 - if maj != cbg.MajArray { 4918 4918 - return fmt.Errorf("expected cbor array") 4919 4919 - } 4920 4920 - 4921 4921 - if extra > 0 { 4922 4922 - t.Dependencies = make([]*Pipeline_Dependency, extra) 4923 4923 - } 4924 4924 - 4925 4925 - for i := 0; i < int(extra); i++ { 4926 4926 - { 4927 4927 - var maj byte 4928 4928 - var extra uint64 4929 4929 - var err error 4930 4930 - _ = maj 4931 4931 - _ = extra 4932 4932 - _ = err 4933 4933 - 4934 4934 - { 4935 4935 - 4936 4936 - b, err := cr.ReadByte() 4937 4937 - if err != nil { 4938 4938 - return err 4939 4939 - } 4940 4940 - if b != cbg.CborNull[0] { 4941 4941 - if err := cr.UnreadByte(); err != nil { 4942 4942 - return err 4943 4943 - } 4944 4944 - t.Dependencies[i] = new(Pipeline_Dependency) 4945 4945 - if err := t.Dependencies[i].UnmarshalCBOR(cr); err != nil { 4946 4946 - return xerrors.Errorf("unmarshaling t.Dependencies[i] pointer: %w", err) 4947 4947 - } 4948 4948 - } 4949 4949 - 4950 4950 - } 4951 4951 - 4952 4952 - } 4405 4405 + t.Engine = string(sval) 4953 4406 } 4954 4407 4955 4408 default:

+4 -18

api/tangled/tangledpipeline.go

··· 29 29 Submodules bool `json:"submodules" cborgen:"submodules"` 30 30 } 31 31 32 32 - // Pipeline_Dependency is a "dependency" in the sh.tangled.pipeline schema. 33 33 - type Pipeline_Dependency struct { 34 34 - Packages []string `json:"packages" cborgen:"packages"` 35 35 - Registry string `json:"registry" cborgen:"registry"` 36 36 - } 37 37 - 38 32 // Pipeline_ManualTriggerData is a "manualTriggerData" in the sh.tangled.pipeline schema. 39 33 type Pipeline_ManualTriggerData struct { 40 34 Inputs []*Pipeline_Pair `json:"inputs,omitempty" cborgen:"inputs,omitempty"` ··· 61 55 Ref string `json:"ref" cborgen:"ref"` 62 56 } 63 57 64 64 - // Pipeline_Step is a "step" in the sh.tangled.pipeline schema. 65 65 - type Pipeline_Step struct { 66 66 - Command string `json:"command" cborgen:"command"` 67 67 - Environment []*Pipeline_Pair `json:"environment,omitempty" cborgen:"environment,omitempty"` 68 68 - Name string `json:"name" cborgen:"name"` 69 69 - } 70 70 - 71 58 // Pipeline_TriggerMetadata is a "triggerMetadata" in the sh.tangled.pipeline schema. 72 59 type Pipeline_TriggerMetadata struct { 73 60 Kind string `json:"kind" cborgen:"kind"` ··· 87 74 88 75 // Pipeline_Workflow is a "workflow" in the sh.tangled.pipeline schema. 89 76 type Pipeline_Workflow struct { 90 90 - Clone *Pipeline_CloneOpts `json:"clone" cborgen:"clone"` 91 91 - Dependencies []*Pipeline_Dependency `json:"dependencies" cborgen:"dependencies"` 92 92 - Environment []*Pipeline_Pair `json:"environment" cborgen:"environment"` 93 93 - Name string `json:"name" cborgen:"name"` 94 94 - Steps []*Pipeline_Step `json:"steps" cborgen:"steps"` 77 77 + Clone *Pipeline_CloneOpts `json:"clone" cborgen:"clone"` 78 78 + Engine string `json:"engine" cborgen:"engine"` 79 79 + Name string `json:"name" cborgen:"name"` 80 80 + Raw string `json:"raw" cborgen:"raw"` 95 81 }

-2

cmd/gen.go

··· 27 27 tangled.KnotMember{}, 28 28 tangled.Pipeline{}, 29 29 tangled.Pipeline_CloneOpts{}, 30 30 - tangled.Pipeline_Dependency{}, 31 30 tangled.Pipeline_ManualTriggerData{}, 32 31 tangled.Pipeline_Pair{}, 33 32 tangled.Pipeline_PullRequestTriggerData{}, 34 33 tangled.Pipeline_PushTriggerData{}, 35 34 tangled.PipelineStatus{}, 36 36 - tangled.Pipeline_Step{}, 37 35 tangled.Pipeline_TriggerMetadata{}, 38 36 tangled.Pipeline_TriggerRepo{}, 39 37 tangled.Pipeline_Workflow{},

+26 -3

docs/spindle/pipeline.md

··· 4 4 repo. Generally: 5 5 6 6 * Pipelines are defined in YAML. 7 7 - * Dependencies can be specified from 8 8 - [Nixpkgs](https://search.nixos.org) or custom registries. 9 9 - * Environment variables can be set globally or per-step. 7 7 + * Workflows can run using different *engines*. 8 8 + 9 9 + The most barebones workflow looks like this: 10 10 + 11 11 + ```yaml 12 12 + when: 13 13 + - event: ["push"] 14 14 + branch: ["main"] 15 15 + 16 16 + engine: "nixery" 17 17 + 18 18 + # optional 19 19 + clone: 20 20 + skip: false 21 21 + depth: 50 22 22 + submodules: true 23 23 + ``` 24 24 + 25 25 + The `when` and `engine` fields are required, while every other aspect 26 26 + of how the definition is parsed is up to the engine. Currently, a spindle 27 27 + provides at least one of these built-in engines: 28 28 + 29 29 + ## `nixery` 30 30 + 31 31 + The Nixery engine uses an instance of [Nixery](https://nixery.dev) to run 32 32 + steps that use dependencies from [Nixpkgs](https://github.com/NixOS/nixpkgs). 10 33 11 34 Here's an example that uses all fields: 12 35

+7 -63

lexicons/pipeline/pipeline.json

··· 149 149 "type": "object", 150 150 "required": [ 151 151 "name", 152 152 - "dependencies", 153 153 - "steps", 154 154 - "environment", 155 155 - "clone" 152 152 + "engine", 153 153 + "clone", 154 154 + "raw" 156 155 ], 157 156 "properties": { 158 157 "name": { 159 158 "type": "string" 160 159 }, 161 161 - "dependencies": { 162 162 - "type": "array", 163 163 - "items": { 164 164 - "type": "ref", 165 165 - "ref": "#dependency" 166 166 - } 167 167 - }, 168 168 - "steps": { 169 169 - "type": "array", 170 170 - "items": { 171 171 - "type": "ref", 172 172 - "ref": "#step" 173 173 - } 174 174 - }, 175 175 - "environment": { 176 176 - "type": "array", 177 177 - "items": { 178 178 - "type": "ref", 179 179 - "ref": "#pair" 180 180 - } 160 160 + "engine": { 161 161 + "type": "string" 181 162 }, 182 163 "clone": { 183 164 "type": "ref", 184 165 "ref": "#cloneOpts" 185 185 - } 186 186 - } 187 187 - }, 188 188 - "dependency": { 189 189 - "type": "object", 190 190 - "required": [ 191 191 - "registry", 192 192 - "packages" 193 193 - ], 194 194 - "properties": { 195 195 - "registry": { 166 166 + }, 167 167 + "raw": { 196 168 "type": "string" 197 197 - }, 198 198 - "packages": { 199 199 - "type": "array", 200 200 - "items": { 201 201 - "type": "string" 202 202 - } 203 169 } 204 170 } 205 171 }, ··· 219 185 }, 220 186 "submodules": { 221 187 "type": "boolean" 222 222 - } 223 223 - } 224 224 - }, 225 225 - "step": { 226 226 - "type": "object", 227 227 - "required": [ 228 228 - "name", 229 229 - "command" 230 230 - ], 231 231 - "properties": { 232 232 - "name": { 233 233 - "type": "string" 234 234 - }, 235 235 - "command": { 236 236 - "type": "string" 237 237 - }, 238 238 - "environment": { 239 239 - "type": "array", 240 240 - "items": { 241 241 - "type": "ref", 242 242 - "ref": "#pair" 243 243 - } 244 188 } 245 189 } 246 190 },

+2 -2

nix/modules/spindle.nix

··· 111 111 "SPINDLE_SERVER_SECRETS_PROVIDER=${cfg.server.secrets.provider}" 112 112 "SPINDLE_SERVER_SECRETS_OPENBAO_PROXY_ADDR=${cfg.server.secrets.openbao.proxyAddr}" 113 113 "SPINDLE_SERVER_SECRETS_OPENBAO_MOUNT=${cfg.server.secrets.openbao.mount}" 114 114 - "SPINDLE_PIPELINES_NIXERY=${cfg.pipelines.nixery}" 115 115 - "SPINDLE_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}" 114 114 + "SPINDLE_NIXERY_PIPELINES_NIXERY=${cfg.pipelines.nixery}" 115 115 + "SPINDLE_NIXERY_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}" 116 116 ]; 117 117 ExecStart = "${cfg.package}/bin/spindle"; 118 118 Restart = "always";

+4 -4

spindle/config/config.go

··· 16 16 Dev bool `env:"DEV, default=false"` 17 17 Owner string `env:"OWNER, required"` 18 18 Secrets Secrets `env:",prefix=SECRETS_"` 19 19 + LogDir string `env:"LOG_DIR, default=/var/log/spindle"` 19 20 } 20 21 21 22 func (s Server) Did() syntax.DID { ··· 32 33 Mount string `env:"MOUNT, default=spindle"` 33 34 } 34 35 35 35 - type Pipelines struct { 36 36 + type NixeryPipelines struct { 36 37 Nixery string `env:"NIXERY, default=nixery.tangled.sh"` 37 38 WorkflowTimeout string `env:"WORKFLOW_TIMEOUT, default=5m"` 38 38 - LogDir string `env:"LOG_DIR, default=/var/log/spindle"` 39 39 } 40 40 41 41 type Config struct { 42 42 - Server Server `env:",prefix=SPINDLE_SERVER_"` 43 43 - Pipelines Pipelines `env:",prefix=SPINDLE_PIPELINES_"` 42 42 + Server Server `env:",prefix=SPINDLE_SERVER_"` 43 43 + NixeryPipelines NixeryPipelines `env:",prefix=SPINDLE_NIXERY_PIPELINES_"` 44 44 } 45 45 46 46 func Load(ctx context.Context) (*Config, error) {

+1 -1

spindle/engine/ansi_stripper.go spindle/engines/nixery/ansi_stripper.go

··· 1 1 - package engine 1 1 + package nixery 2 2 3 3 import ( 4 4 "io"

+64 -416

spindle/engine/engine.go

··· 4 4 "context" 5 5 "errors" 6 6 "fmt" 7 7 - "io" 8 7 "log/slog" 9 9 - "os" 10 10 - "strings" 11 11 - "sync" 12 12 - "time" 13 8 14 9 securejoin "github.com/cyphar/filepath-securejoin" 15 15 - "github.com/docker/docker/api/types/container" 16 16 - "github.com/docker/docker/api/types/image" 17 17 - "github.com/docker/docker/api/types/mount" 18 18 - "github.com/docker/docker/api/types/network" 19 19 - "github.com/docker/docker/api/types/volume" 20 20 - "github.com/docker/docker/client" 21 21 - "github.com/docker/docker/pkg/stdcopy" 22 10 "golang.org/x/sync/errgroup" 23 23 - "tangled.sh/tangled.sh/core/log" 24 11 "tangled.sh/tangled.sh/core/notifier" 25 12 "tangled.sh/tangled.sh/core/spindle/config" 26 13 "tangled.sh/tangled.sh/core/spindle/db" ··· 28 15 "tangled.sh/tangled.sh/core/spindle/secrets" 29 16 ) 30 17 31 31 - const ( 32 32 - workspaceDir = "/tangled/workspace" 18 18 + var ( 19 19 + ErrTimedOut = errors.New("timed out") 20 20 + ErrWorkflowFailed = errors.New("workflow failed") 33 21 ) 34 22 35 35 - type cleanupFunc func(context.Context) error 36 36 - 37 37 - type Engine struct { 38 38 - docker client.APIClient 39 39 - l *slog.Logger 40 40 - db *db.DB 41 41 - n *notifier.Notifier 42 42 - cfg *config.Config 43 43 - vault secrets.Manager 44 44 - 45 45 - cleanupMu sync.Mutex 46 46 - cleanup map[string][]cleanupFunc 47 47 - } 48 48 - 49 49 - func New(ctx context.Context, cfg *config.Config, db *db.DB, n *notifier.Notifier, vault secrets.Manager) (*Engine, error) { 50 50 - dcli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) 51 51 - if err != nil { 52 52 - return nil, err 53 53 - } 54 54 - 55 55 - l := log.FromContext(ctx).With("component", "spindle") 56 56 - 57 57 - e := &Engine{ 58 58 - docker: dcli, 59 59 - l: l, 60 60 - db: db, 61 61 - n: n, 62 62 - cfg: cfg, 63 63 - vault: vault, 64 64 - } 65 65 - 66 66 - e.cleanup = make(map[string][]cleanupFunc) 67 67 - 68 68 - return e, nil 69 69 - } 70 70 - 71 71 - func (e *Engine) StartWorkflows(ctx context.Context, pipeline *models.Pipeline, pipelineId models.PipelineId) { 72 72 - e.l.Info("starting all workflows in parallel", "pipeline", pipelineId) 23 23 + func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, db *db.DB, n *notifier.Notifier, ctx context.Context, pipeline *models.Pipeline, pipelineId models.PipelineId) { 24 24 + l.Info("starting all workflows in parallel", "pipeline", pipelineId) 73 25 74 26 // extract secrets 75 27 var allSecrets []secrets.UnlockedSecret 76 28 if didSlashRepo, err := securejoin.SecureJoin(pipeline.RepoOwner, pipeline.RepoName); err == nil { 77 77 - if res, err := e.vault.GetSecretsUnlocked(ctx, secrets.DidSlashRepo(didSlashRepo)); err == nil { 29 29 + if res, err := vault.GetSecretsUnlocked(ctx, secrets.DidSlashRepo(didSlashRepo)); err == nil { 78 30 allSecrets = res 79 31 } 80 32 } 81 33 82 82 - workflowTimeoutStr := e.cfg.Pipelines.WorkflowTimeout 83 83 - workflowTimeout, err := time.ParseDuration(workflowTimeoutStr) 84 84 - if err != nil { 85 85 - e.l.Error("failed to parse workflow timeout", "error", err, "timeout", workflowTimeoutStr) 86 86 - workflowTimeout = 5 * time.Minute 87 87 - } 88 88 - e.l.Info("using workflow timeout", "timeout", workflowTimeout) 89 89 - 90 34 eg, ctx := errgroup.WithContext(ctx) 91 91 - for _, w := range pipeline.Workflows { 92 92 - eg.Go(func() error { 93 93 - wid := models.WorkflowId{ 94 94 - PipelineId: pipelineId, 95 95 - Name: w.Name, 96 96 - } 97 97 - 98 98 - err := e.db.StatusRunning(wid, e.n) 99 99 - if err != nil { 100 100 - return err 101 101 - } 35 35 + for eng, wfs := range pipeline.Workflows { 36 36 + workflowTimeout := eng.WorkflowTimeout() 37 37 + l.Info("using workflow timeout", "timeout", workflowTimeout) 102 38 103 103 - err = e.SetupWorkflow(ctx, wid) 104 104 - if err != nil { 105 105 - e.l.Error("setting up worklow", "wid", wid, "err", err) 106 106 - return err 107 107 - } 108 108 - defer e.DestroyWorkflow(ctx, wid) 109 109 - 110 110 - reader, err := e.docker.ImagePull(ctx, w.Image, image.PullOptions{}) 111 111 - if err != nil { 112 112 - e.l.Error("pipeline image pull failed!", "image", w.Image, "workflowId", wid, "error", err.Error()) 39 39 + for _, w := range wfs { 40 40 + eg.Go(func() error { 41 41 + wid := models.WorkflowId{ 42 42 + PipelineId: pipelineId, 43 43 + Name: w.Name, 44 44 + } 113 45 114 114 - err := e.db.StatusFailed(wid, err.Error(), -1, e.n) 46 46 + err := db.StatusRunning(wid, n) 115 47 if err != nil { 116 48 return err 117 49 } 118 50 119 119 - return fmt.Errorf("pulling image: %w", err) 120 120 - } 121 121 - defer reader.Close() 122 122 - io.Copy(os.Stdout, reader) 123 123 - 124 124 - ctx, cancel := context.WithTimeout(ctx, workflowTimeout) 125 125 - defer cancel() 51 51 + err = eng.SetupWorkflow(ctx, wid, &w) 52 52 + if err != nil { 53 53 + // TODO(winter): Should this always set StatusFailed? 54 54 + // In the original, we only do in a subset of cases. 55 55 + l.Error("setting up worklow", "wid", wid, "err", err) 126 56 127 127 - err = e.StartSteps(ctx, wid, w, allSecrets) 128 128 - if err != nil { 129 129 - if errors.Is(err, ErrTimedOut) { 130 130 - dbErr := e.db.StatusTimeout(wid, e.n) 131 131 - if dbErr != nil { 132 132 - return dbErr 133 133 - } 134 134 - } else { 135 135 - dbErr := e.db.StatusFailed(wid, err.Error(), -1, e.n) 57 57 + dbErr := db.StatusFailed(wid, err.Error(), -1, n) 136 58 if dbErr != nil { 137 59 return dbErr 138 60 } 61 61 + return err 139 62 } 140 140 - 141 141 - return fmt.Errorf("starting steps image: %w", err) 142 142 - } 143 143 - 144 144 - err = e.db.StatusSuccess(wid, e.n) 145 145 - if err != nil { 146 146 - return err 147 147 - } 148 148 - 149 149 - return nil 150 150 - }) 151 151 - } 152 152 - 153 153 - if err = eg.Wait(); err != nil { 154 154 - e.l.Error("failed to run one or more workflows", "err", err) 155 155 - } else { 156 156 - e.l.Error("successfully ran full pipeline") 157 157 - } 158 158 - } 159 159 - 160 160 - // SetupWorkflow sets up a new network for the workflow and volumes for 161 161 - // the workspace and Nix store. These are persisted across steps and are 162 162 - // destroyed at the end of the workflow. 163 163 - func (e *Engine) SetupWorkflow(ctx context.Context, wid models.WorkflowId) error { 164 164 - e.l.Info("setting up workflow", "workflow", wid) 165 165 - 166 166 - _, err := e.docker.VolumeCreate(ctx, volume.CreateOptions{ 167 167 - Name: workspaceVolume(wid), 168 168 - Driver: "local", 169 169 - }) 170 170 - if err != nil { 171 171 - return err 172 172 - } 173 173 - e.registerCleanup(wid, func(ctx context.Context) error { 174 174 - return e.docker.VolumeRemove(ctx, workspaceVolume(wid), true) 175 175 - }) 176 176 - 177 177 - _, err = e.docker.VolumeCreate(ctx, volume.CreateOptions{ 178 178 - Name: nixVolume(wid), 179 179 - Driver: "local", 180 180 - }) 181 181 - if err != nil { 182 182 - return err 183 183 - } 184 184 - e.registerCleanup(wid, func(ctx context.Context) error { 185 185 - return e.docker.VolumeRemove(ctx, nixVolume(wid), true) 186 186 - }) 187 187 - 188 188 - _, err = e.docker.NetworkCreate(ctx, networkName(wid), network.CreateOptions{ 189 189 - Driver: "bridge", 190 190 - }) 191 191 - if err != nil { 192 192 - return err 193 193 - } 194 194 - e.registerCleanup(wid, func(ctx context.Context) error { 195 195 - return e.docker.NetworkRemove(ctx, networkName(wid)) 196 196 - }) 197 197 - 198 198 - return nil 199 199 - } 200 200 - 201 201 - // StartSteps starts all steps sequentially with the same base image. 202 202 - // ONLY marks pipeline as failed if container's exit code is non-zero. 203 203 - // All other errors are bubbled up. 204 204 - // Fixed version of the step execution logic 205 205 - func (e *Engine) StartSteps(ctx context.Context, wid models.WorkflowId, w models.Workflow, secrets []secrets.UnlockedSecret) error { 206 206 - workflowEnvs := ConstructEnvs(w.Environment) 207 207 - for _, s := range secrets { 208 208 - workflowEnvs.AddEnv(s.Key, s.Value) 209 209 - } 210 210 - 211 211 - for stepIdx, step := range w.Steps { 212 212 - select { 213 213 - case <-ctx.Done(): 214 214 - return ctx.Err() 215 215 - default: 216 216 - } 217 217 - 218 218 - envs := append(EnvVars(nil), workflowEnvs...) 219 219 - for k, v := range step.Environment { 220 220 - envs.AddEnv(k, v) 221 221 - } 222 222 - envs.AddEnv("HOME", workspaceDir) 223 223 - e.l.Debug("envs for step", "step", step.Name, "envs", envs.Slice()) 224 224 - 225 225 - hostConfig := hostConfig(wid) 226 226 - resp, err := e.docker.ContainerCreate(ctx, &container.Config{ 227 227 - Image: w.Image, 228 228 - Cmd: []string{"bash", "-c", step.Command}, 229 229 - WorkingDir: workspaceDir, 230 230 - Tty: false, 231 231 - Hostname: "spindle", 232 232 - Env: envs.Slice(), 233 233 - }, hostConfig, nil, nil, "") 234 234 - defer e.DestroyStep(ctx, resp.ID) 235 235 - if err != nil { 236 236 - return fmt.Errorf("creating container: %w", err) 237 237 - } 238 238 - 239 239 - err = e.docker.NetworkConnect(ctx, networkName(wid), resp.ID, nil) 240 240 - if err != nil { 241 241 - return fmt.Errorf("connecting network: %w", err) 242 242 - } 243 243 - 244 244 - err = e.docker.ContainerStart(ctx, resp.ID, container.StartOptions{}) 245 245 - if err != nil { 246 246 - return err 247 247 - } 248 248 - e.l.Info("started container", "name", resp.ID, "step", step.Name) 249 249 - 250 250 - // start tailing logs in background 251 251 - tailDone := make(chan error, 1) 252 252 - go func() { 253 253 - tailDone <- e.TailStep(ctx, resp.ID, wid, stepIdx, step) 254 254 - }() 255 255 - 256 256 - // wait for container completion or timeout 257 257 - waitDone := make(chan struct{}) 258 258 - var state *container.State 259 259 - var waitErr error 260 260 - 261 261 - go func() { 262 262 - defer close(waitDone) 263 263 - state, waitErr = e.WaitStep(ctx, resp.ID) 264 264 - }() 265 265 - 266 266 - select { 267 267 - case <-waitDone: 268 268 - 269 269 - // wait for tailing to complete 270 270 - <-tailDone 271 271 - 272 272 - case <-ctx.Done(): 273 273 - e.l.Warn("step timed out; killing container", "container", resp.ID, "step", step.Name) 274 274 - err = e.DestroyStep(context.Background(), resp.ID) 275 275 - if err != nil { 276 276 - e.l.Error("failed to destroy step", "container", resp.ID, "error", err) 277 277 - } 278 278 - 279 279 - // wait for both goroutines to finish 280 280 - <-waitDone 281 281 - <-tailDone 282 282 - 283 283 - return ErrTimedOut 284 284 - } 285 285 - 286 286 - select { 287 287 - case <-ctx.Done(): 288 288 - return ctx.Err() 289 289 - default: 290 290 - } 291 291 - 292 292 - if waitErr != nil { 293 293 - return waitErr 294 294 - } 295 295 - 296 296 - err = e.DestroyStep(ctx, resp.ID) 297 297 - if err != nil { 298 298 - return err 299 299 - } 300 300 - 301 301 - if state.ExitCode != 0 { 302 302 - e.l.Error("workflow failed!", "workflow_id", wid.String(), "error", state.Error, "exit_code", state.ExitCode, "oom_killed", state.OOMKilled) 303 303 - if state.OOMKilled { 304 304 - return ErrOOMKilled 305 305 - } 306 306 - return ErrWorkflowFailed 307 307 - } 308 308 - } 309 309 - 310 310 - return nil 311 311 - } 312 312 - 313 313 - func (e *Engine) WaitStep(ctx context.Context, containerID string) (*container.State, error) { 314 314 - wait, errCh := e.docker.ContainerWait(ctx, containerID, container.WaitConditionNotRunning) 315 315 - select { 316 316 - case err := <-errCh: 317 317 - if err != nil { 318 318 - return nil, err 319 319 - } 320 320 - case <-wait: 321 321 - } 322 322 - 323 323 - e.l.Info("waited for container", "name", containerID) 324 324 - 325 325 - info, err := e.docker.ContainerInspect(ctx, containerID) 326 326 - if err != nil { 327 327 - return nil, err 328 328 - } 329 329 - 330 330 - return info.State, nil 331 331 - } 332 332 - 333 333 - func (e *Engine) TailStep(ctx context.Context, containerID string, wid models.WorkflowId, stepIdx int, step models.Step) error { 334 334 - wfLogger, err := NewWorkflowLogger(e.cfg.Pipelines.LogDir, wid) 335 335 - if err != nil { 336 336 - e.l.Warn("failed to setup step logger; logs will not be persisted", "error", err) 337 337 - return err 338 338 - } 339 339 - defer wfLogger.Close() 340 340 - 341 341 - ctl := wfLogger.ControlWriter(stepIdx, step) 342 342 - ctl.Write([]byte(step.Name)) 343 343 - 344 344 - logs, err := e.docker.ContainerLogs(ctx, containerID, container.LogsOptions{ 345 345 - Follow: true, 346 346 - ShowStdout: true, 347 347 - ShowStderr: true, 348 348 - Details: false, 349 349 - Timestamps: false, 350 350 - }) 351 351 - if err != nil { 352 352 - return err 353 353 - } 354 354 - 355 355 - _, err = stdcopy.StdCopy( 356 356 - wfLogger.DataWriter("stdout"), 357 357 - wfLogger.DataWriter("stderr"), 358 358 - logs, 359 359 - ) 360 360 - if err != nil && err != io.EOF && !errors.Is(err, context.DeadlineExceeded) { 361 361 - return fmt.Errorf("failed to copy logs: %w", err) 362 362 - } 63 63 + defer eng.DestroyWorkflow(ctx, wid) 363 64 364 364 - return nil 365 365 - } 65 65 + wfLogger, err := models.NewWorkflowLogger(cfg.Server.LogDir, wid) 66 66 + if err != nil { 67 67 + l.Warn("failed to setup step logger; logs will not be persisted", "error", err) 68 68 + wfLogger = nil 69 69 + } else { 70 70 + defer wfLogger.Close() 71 71 + } 366 72 367 367 - func (e *Engine) DestroyStep(ctx context.Context, containerID string) error { 368 368 - err := e.docker.ContainerKill(ctx, containerID, "9") // SIGKILL 369 369 - if err != nil && !isErrContainerNotFoundOrNotRunning(err) { 370 370 - return err 371 371 - } 73 73 + ctx, cancel := context.WithTimeout(ctx, workflowTimeout) 74 74 + defer cancel() 372 75 373 373 - if err := e.docker.ContainerRemove(ctx, containerID, container.RemoveOptions{ 374 374 - RemoveVolumes: true, 375 375 - RemoveLinks: false, 376 376 - Force: false, 377 377 - }); err != nil && !isErrContainerNotFoundOrNotRunning(err) { 378 378 - return err 379 379 - } 76 76 + for stepIdx, step := range w.Steps { 77 77 + if wfLogger != nil { 78 78 + ctl := wfLogger.ControlWriter(stepIdx, step) 79 79 + ctl.Write([]byte(step.Name())) 80 80 + } 380 81 381 381 - return nil 382 382 - } 82 82 + err = eng.RunStep(ctx, wid, &w, stepIdx, allSecrets, wfLogger) 83 83 + if err != nil { 84 84 + if errors.Is(err, ErrTimedOut) { 85 85 + dbErr := db.StatusTimeout(wid, n) 86 86 + if dbErr != nil { 87 87 + return dbErr 88 88 + } 89 89 + } else { 90 90 + dbErr := db.StatusFailed(wid, err.Error(), -1, n) 91 91 + if dbErr != nil { 92 92 + return dbErr 93 93 + } 94 94 + } 383 95 384 384 - func (e *Engine) DestroyWorkflow(ctx context.Context, wid models.WorkflowId) error { 385 385 - e.cleanupMu.Lock() 386 386 - key := wid.String() 96 96 + return fmt.Errorf("starting steps image: %w", err) 97 97 + } 98 98 + } 387 99 388 388 - fns := e.cleanup[key] 389 389 - delete(e.cleanup, key) 390 390 - e.cleanupMu.Unlock() 100 100 + err = db.StatusSuccess(wid, n) 101 101 + if err != nil { 102 102 + return err 103 103 + } 391 104 392 392 - for _, fn := range fns { 393 393 - if err := fn(ctx); err != nil { 394 394 - e.l.Error("failed to cleanup workflow resource", "workflowId", wid, "error", err) 105 105 + return nil 106 106 + }) 395 107 } 396 108 } 397 397 - return nil 398 398 - } 399 109 400 400 - func (e *Engine) registerCleanup(wid models.WorkflowId, fn cleanupFunc) { 401 401 - e.cleanupMu.Lock() 402 402 - defer e.cleanupMu.Unlock() 403 403 - 404 404 - key := wid.String() 405 405 - e.cleanup[key] = append(e.cleanup[key], fn) 406 406 - } 407 407 - 408 408 - func workspaceVolume(wid models.WorkflowId) string { 409 409 - return fmt.Sprintf("workspace-%s", wid) 410 410 - } 411 411 - 412 412 - func nixVolume(wid models.WorkflowId) string { 413 413 - return fmt.Sprintf("nix-%s", wid) 414 414 - } 415 415 - 416 416 - func networkName(wid models.WorkflowId) string { 417 417 - return fmt.Sprintf("workflow-network-%s", wid) 418 418 - } 419 419 - 420 420 - func hostConfig(wid models.WorkflowId) *container.HostConfig { 421 421 - hostConfig := &container.HostConfig{ 422 422 - Mounts: []mount.Mount{ 423 423 - { 424 424 - Type: mount.TypeVolume, 425 425 - Source: workspaceVolume(wid), 426 426 - Target: workspaceDir, 427 427 - }, 428 428 - { 429 429 - Type: mount.TypeVolume, 430 430 - Source: nixVolume(wid), 431 431 - Target: "/nix", 432 432 - }, 433 433 - { 434 434 - Type: mount.TypeTmpfs, 435 435 - Target: "/tmp", 436 436 - ReadOnly: false, 437 437 - TmpfsOptions: &mount.TmpfsOptions{ 438 438 - Mode: 0o1777, // world-writeable sticky bit 439 439 - Options: [][]string{ 440 440 - {"exec"}, 441 441 - }, 442 442 - }, 443 443 - }, 444 444 - { 445 445 - Type: mount.TypeVolume, 446 446 - Source: "etc-nix-" + wid.String(), 447 447 - Target: "/etc/nix", 448 448 - }, 449 449 - }, 450 450 - ReadonlyRootfs: false, 451 451 - CapDrop: []string{"ALL"}, 452 452 - CapAdd: []string{"CAP_DAC_OVERRIDE"}, 453 453 - SecurityOpt: []string{"no-new-privileges"}, 454 454 - ExtraHosts: []string{"host.docker.internal:host-gateway"}, 110 110 + if err := eg.Wait(); err != nil { 111 111 + l.Error("failed to run one or more workflows", "err", err) 112 112 + } else { 113 113 + l.Error("successfully ran full pipeline") 455 114 } 456 456 - 457 457 - return hostConfig 458 458 - } 459 459 - 460 460 - // thanks woodpecker 461 461 - func isErrContainerNotFoundOrNotRunning(err error) bool { 462 462 - // Error response from daemon: Cannot kill container: ...: No such container: ... 463 463 - // Error response from daemon: Cannot kill container: ...: Container ... is not running" 464 464 - // Error response from podman daemon: can only kill running containers. ... is in state exited 465 465 - // Error: No such container: ... 466 466 - return err != nil && (strings.Contains(err.Error(), "No such container") || strings.Contains(err.Error(), "is not running") || strings.Contains(err.Error(), "can only kill running containers")) 467 115 }

+1 -1

spindle/engine/envs.go spindle/engines/nixery/envs.go

··· 1 1 - package engine 1 1 + package nixery 2 2 3 3 import ( 4 4 "fmt"

+1 -1

spindle/engine/envs_test.go spindle/engines/nixery/envs_test.go

··· 1 1 - package engine 1 1 + package nixery 2 2 3 3 import ( 4 4 "testing"

-9

spindle/engine/errors.go

··· 1 1 - package engine 2 2 - 3 3 - import "errors" 4 4 - 5 5 - var ( 6 6 - ErrOOMKilled = errors.New("oom killed") 7 7 - ErrTimedOut = errors.New("timed out") 8 8 - ErrWorkflowFailed = errors.New("workflow failed") 9 9 - )

+8 -10

spindle/engine/logger.go spindle/models/logger.go

··· 1 1 - package engine 1 1 + package models 2 2 3 3 import ( 4 4 "encoding/json" ··· 7 7 "os" 8 8 "path/filepath" 9 9 "strings" 10 10 - 11 11 - "tangled.sh/tangled.sh/core/spindle/models" 12 10 ) 13 11 14 12 type WorkflowLogger struct { ··· 16 14 encoder *json.Encoder 17 15 } 18 16 19 19 - func NewWorkflowLogger(baseDir string, wid models.WorkflowId) (*WorkflowLogger, error) { 17 17 + func NewWorkflowLogger(baseDir string, wid WorkflowId) (*WorkflowLogger, error) { 20 18 path := LogFilePath(baseDir, wid) 21 19 22 20 file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) ··· 30 28 }, nil 31 29 } 32 30 33 33 - func LogFilePath(baseDir string, workflowID models.WorkflowId) string { 31 31 + func LogFilePath(baseDir string, workflowID WorkflowId) string { 34 32 logFilePath := filepath.Join(baseDir, fmt.Sprintf("%s.log", workflowID.String())) 35 33 return logFilePath 36 34 } ··· 47 45 } 48 46 } 49 47 50 50 - func (l *WorkflowLogger) ControlWriter(idx int, step models.Step) io.Writer { 48 48 + func (l *WorkflowLogger) ControlWriter(idx int, step Step) io.Writer { 51 49 return &controlWriter{ 52 50 logger: l, 53 51 idx: idx, ··· 62 60 63 61 func (w *dataWriter) Write(p []byte) (int, error) { 64 62 line := strings.TrimRight(string(p), "\r\n") 65 65 - entry := models.NewDataLogLine(line, w.stream) 63 63 + entry := NewDataLogLine(line, w.stream) 66 64 if err := w.logger.encoder.Encode(entry); err != nil { 67 65 return 0, err 68 66 } ··· 72 70 type controlWriter struct { 73 71 logger *WorkflowLogger 74 72 idx int 75 75 - step models.Step 73 73 + step Step 76 74 } 77 75 78 76 func (w *controlWriter) Write(_ []byte) (int, error) { 79 79 - entry := models.NewControlLogLine(w.idx, w.step) 77 77 + entry := NewControlLogLine(w.idx, w.step) 80 78 if err := w.logger.encoder.Encode(entry); err != nil { 81 79 return 0, err 82 80 } 83 83 - return len(w.step.Name), nil 81 81 + return len(w.step.Name()), nil 84 82 }

+476

spindle/engines/nixery/engine.go

··· 1 1 + package nixery 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "errors" 6 6 + "fmt" 7 7 + "io" 8 8 + "log/slog" 9 9 + "os" 10 10 + "path" 11 11 + "strings" 12 12 + "sync" 13 13 + "time" 14 14 + 15 15 + "github.com/docker/docker/api/types/container" 16 16 + "github.com/docker/docker/api/types/image" 17 17 + "github.com/docker/docker/api/types/mount" 18 18 + "github.com/docker/docker/api/types/network" 19 19 + "github.com/docker/docker/api/types/volume" 20 20 + "github.com/docker/docker/client" 21 21 + "github.com/docker/docker/pkg/stdcopy" 22 22 + "gopkg.in/yaml.v3" 23 23 + "tangled.sh/tangled.sh/core/api/tangled" 24 24 + "tangled.sh/tangled.sh/core/log" 25 25 + "tangled.sh/tangled.sh/core/spindle/config" 26 26 + "tangled.sh/tangled.sh/core/spindle/engine" 27 27 + "tangled.sh/tangled.sh/core/spindle/models" 28 28 + "tangled.sh/tangled.sh/core/spindle/secrets" 29 29 + ) 30 30 + 31 31 + const ( 32 32 + workspaceDir = "/tangled/workspace" 33 33 + ) 34 34 + 35 35 + type cleanupFunc func(context.Context) error 36 36 + 37 37 + type Engine struct { 38 38 + docker client.APIClient 39 39 + l *slog.Logger 40 40 + cfg *config.Config 41 41 + 42 42 + cleanupMu sync.Mutex 43 43 + cleanup map[string][]cleanupFunc 44 44 + } 45 45 + 46 46 + type Step struct { 47 47 + name string 48 48 + kind models.StepKind 49 49 + command string 50 50 + environment map[string]string 51 51 + } 52 52 + 53 53 + func (s Step) Name() string { 54 54 + return s.name 55 55 + } 56 56 + 57 57 + func (s Step) Command() string { 58 58 + return s.command 59 59 + } 60 60 + 61 61 + func (s Step) Kind() models.StepKind { 62 62 + return s.kind 63 63 + } 64 64 + 65 65 + // setupSteps get added to start of Steps 66 66 + type setupSteps []models.Step 67 67 + 68 68 + // addStep adds a step to the beginning of the workflow's steps. 69 69 + func (ss *setupSteps) addStep(step models.Step) { 70 70 + *ss = append(*ss, step) 71 71 + } 72 72 + 73 73 + type addlFields struct { 74 74 + image string 75 75 + env map[string]string 76 76 + } 77 77 + 78 78 + func (e *Engine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipeline) (*models.Workflow, error) { 79 79 + swf := &models.Workflow{} 80 80 + addl := addlFields{} 81 81 + 82 82 + dwf := &struct { 83 83 + Steps []struct { 84 84 + Command string `yaml:"command"` 85 85 + Name string `yaml:"name"` 86 86 + Environment map[string]string `yaml:"environment"` 87 87 + } `yaml:"steps"` 88 88 + Dependencies map[string][]string `yaml:"dependencies"` 89 89 + Environment map[string]string `yaml:"environment"` 90 90 + }{} 91 91 + err := yaml.Unmarshal([]byte(twf.Raw), &dwf) 92 92 + if err != nil { 93 93 + return nil, err 94 94 + } 95 95 + 96 96 + for _, dstep := range dwf.Steps { 97 97 + sstep := Step{} 98 98 + sstep.environment = dstep.Environment 99 99 + sstep.command = dstep.Command 100 100 + sstep.name = dstep.Name 101 101 + sstep.kind = models.StepKindUser 102 102 + swf.Steps = append(swf.Steps, sstep) 103 103 + } 104 104 + swf.Name = twf.Name 105 105 + addl.env = dwf.Environment 106 106 + addl.image = workflowImage(dwf.Dependencies, e.cfg.NixeryPipelines.Nixery) 107 107 + 108 108 + setup := &setupSteps{} 109 109 + 110 110 + setup.addStep(nixConfStep()) 111 111 + setup.addStep(cloneStep(twf, *tpl.TriggerMetadata, e.cfg.Server.Dev)) 112 112 + // this step could be empty 113 113 + if s := dependencyStep(dwf.Dependencies); s != nil { 114 114 + setup.addStep(*s) 115 115 + } 116 116 + 117 117 + // append setup steps in order to the start of workflow steps 118 118 + swf.Steps = append(*setup, swf.Steps...) 119 119 + swf.Data = addl 120 120 + 121 121 + return swf, nil 122 122 + } 123 123 + 124 124 + func (e *Engine) WorkflowTimeout() time.Duration { 125 125 + workflowTimeoutStr := e.cfg.NixeryPipelines.WorkflowTimeout 126 126 + workflowTimeout, err := time.ParseDuration(workflowTimeoutStr) 127 127 + if err != nil { 128 128 + e.l.Error("failed to parse workflow timeout", "error", err, "timeout", workflowTimeoutStr) 129 129 + workflowTimeout = 5 * time.Minute 130 130 + } 131 131 + 132 132 + return workflowTimeout 133 133 + } 134 134 + 135 135 + func workflowImage(deps map[string][]string, nixery string) string { 136 136 + var dependencies string 137 137 + for reg, ds := range deps { 138 138 + if reg == "nixpkgs" { 139 139 + dependencies = path.Join(ds...) 140 140 + } 141 141 + } 142 142 + 143 143 + // load defaults from somewhere else 144 144 + dependencies = path.Join(dependencies, "bash", "git", "coreutils", "nix") 145 145 + 146 146 + return path.Join(nixery, dependencies) 147 147 + } 148 148 + 149 149 + func New(ctx context.Context, cfg *config.Config) (*Engine, error) { 150 150 + dcli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) 151 151 + if err != nil { 152 152 + return nil, err 153 153 + } 154 154 + 155 155 + l := log.FromContext(ctx).With("component", "spindle") 156 156 + 157 157 + e := &Engine{ 158 158 + docker: dcli, 159 159 + l: l, 160 160 + cfg: cfg, 161 161 + } 162 162 + 163 163 + e.cleanup = make(map[string][]cleanupFunc) 164 164 + 165 165 + return e, nil 166 166 + } 167 167 + 168 168 + // SetupWorkflow sets up a new network for the workflow and volumes for 169 169 + // the workspace and Nix store. These are persisted across steps and are 170 170 + // destroyed at the end of the workflow. 171 171 + func (e *Engine) SetupWorkflow(ctx context.Context, wid models.WorkflowId, wf *models.Workflow) error { 172 172 + e.l.Info("setting up workflow", "workflow", wid) 173 173 + 174 174 + _, err := e.docker.VolumeCreate(ctx, volume.CreateOptions{ 175 175 + Name: workspaceVolume(wid), 176 176 + Driver: "local", 177 177 + }) 178 178 + if err != nil { 179 179 + return err 180 180 + } 181 181 + e.registerCleanup(wid, func(ctx context.Context) error { 182 182 + return e.docker.VolumeRemove(ctx, workspaceVolume(wid), true) 183 183 + }) 184 184 + 185 185 + _, err = e.docker.VolumeCreate(ctx, volume.CreateOptions{ 186 186 + Name: nixVolume(wid), 187 187 + Driver: "local", 188 188 + }) 189 189 + if err != nil { 190 190 + return err 191 191 + } 192 192 + e.registerCleanup(wid, func(ctx context.Context) error { 193 193 + return e.docker.VolumeRemove(ctx, nixVolume(wid), true) 194 194 + }) 195 195 + 196 196 + _, err = e.docker.NetworkCreate(ctx, networkName(wid), network.CreateOptions{ 197 197 + Driver: "bridge", 198 198 + }) 199 199 + if err != nil { 200 200 + return err 201 201 + } 202 202 + e.registerCleanup(wid, func(ctx context.Context) error { 203 203 + return e.docker.NetworkRemove(ctx, networkName(wid)) 204 204 + }) 205 205 + 206 206 + addl := wf.Data.(addlFields) 207 207 + 208 208 + reader, err := e.docker.ImagePull(ctx, addl.image, image.PullOptions{}) 209 209 + if err != nil { 210 210 + e.l.Error("pipeline image pull failed!", "image", addl.image, "workflowId", wid, "error", err.Error()) 211 211 + 212 212 + return fmt.Errorf("pulling image: %w", err) 213 213 + } 214 214 + defer reader.Close() 215 215 + io.Copy(os.Stdout, reader) 216 216 + 217 217 + return nil 218 218 + } 219 219 + 220 220 + func (e *Engine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.Workflow, idx int, secrets []secrets.UnlockedSecret, wfLogger *models.WorkflowLogger) error { 221 221 + workflowEnvs := ConstructEnvs(w.Data.(addlFields).env) 222 222 + for _, s := range secrets { 223 223 + workflowEnvs.AddEnv(s.Key, s.Value) 224 224 + } 225 225 + 226 226 + step := w.Steps[idx].(Step) 227 227 + 228 228 + select { 229 229 + case <-ctx.Done(): 230 230 + return ctx.Err() 231 231 + default: 232 232 + } 233 233 + 234 234 + envs := append(EnvVars(nil), workflowEnvs...) 235 235 + for k, v := range step.environment { 236 236 + envs.AddEnv(k, v) 237 237 + } 238 238 + envs.AddEnv("HOME", workspaceDir) 239 239 + e.l.Debug("envs for step", "step", step.Name, "envs", envs.Slice()) 240 240 + 241 241 + hostConfig := hostConfig(wid) 242 242 + resp, err := e.docker.ContainerCreate(ctx, &container.Config{ 243 243 + Image: w.Data.(addlFields).image, 244 244 + Cmd: []string{"bash", "-c", step.command}, 245 245 + WorkingDir: workspaceDir, 246 246 + Tty: false, 247 247 + Hostname: "spindle", 248 248 + Env: envs.Slice(), 249 249 + }, hostConfig, nil, nil, "") 250 250 + defer e.DestroyStep(ctx, resp.ID) 251 251 + if err != nil { 252 252 + return fmt.Errorf("creating container: %w", err) 253 253 + } 254 254 + 255 255 + err = e.docker.NetworkConnect(ctx, networkName(wid), resp.ID, nil) 256 256 + if err != nil { 257 257 + return fmt.Errorf("connecting network: %w", err) 258 258 + } 259 259 + 260 260 + err = e.docker.ContainerStart(ctx, resp.ID, container.StartOptions{}) 261 261 + if err != nil { 262 262 + return err 263 263 + } 264 264 + e.l.Info("started container", "name", resp.ID, "step", step.Name) 265 265 + 266 266 + // start tailing logs in background 267 267 + tailDone := make(chan error, 1) 268 268 + go func() { 269 269 + tailDone <- e.tailStep(ctx, wfLogger, resp.ID, wid, idx, step) 270 270 + }() 271 271 + 272 272 + // wait for container completion or timeout 273 273 + waitDone := make(chan struct{}) 274 274 + var state *container.State 275 275 + var waitErr error 276 276 + 277 277 + go func() { 278 278 + defer close(waitDone) 279 279 + state, waitErr = e.WaitStep(ctx, resp.ID) 280 280 + }() 281 281 + 282 282 + select { 283 283 + case <-waitDone: 284 284 + 285 285 + // wait for tailing to complete 286 286 + <-tailDone 287 287 + 288 288 + case <-ctx.Done(): 289 289 + e.l.Warn("step timed out; killing container", "container", resp.ID, "step", step.Name) 290 290 + err = e.DestroyStep(context.Background(), resp.ID) 291 291 + if err != nil { 292 292 + e.l.Error("failed to destroy step", "container", resp.ID, "error", err) 293 293 + } 294 294 + 295 295 + // wait for both goroutines to finish 296 296 + <-waitDone 297 297 + <-tailDone 298 298 + 299 299 + return engine.ErrTimedOut 300 300 + } 301 301 + 302 302 + select { 303 303 + case <-ctx.Done(): 304 304 + return ctx.Err() 305 305 + default: 306 306 + } 307 307 + 308 308 + if waitErr != nil { 309 309 + return waitErr 310 310 + } 311 311 + 312 312 + err = e.DestroyStep(ctx, resp.ID) 313 313 + if err != nil { 314 314 + return err 315 315 + } 316 316 + 317 317 + if state.ExitCode != 0 { 318 318 + e.l.Error("workflow failed!", "workflow_id", wid.String(), "error", state.Error, "exit_code", state.ExitCode, "oom_killed", state.OOMKilled) 319 319 + if state.OOMKilled { 320 320 + return ErrOOMKilled 321 321 + } 322 322 + return engine.ErrWorkflowFailed 323 323 + } 324 324 + 325 325 + return nil 326 326 + } 327 327 + 328 328 + func (e *Engine) WaitStep(ctx context.Context, containerID string) (*container.State, error) { 329 329 + wait, errCh := e.docker.ContainerWait(ctx, containerID, container.WaitConditionNotRunning) 330 330 + select { 331 331 + case err := <-errCh: 332 332 + if err != nil { 333 333 + return nil, err 334 334 + } 335 335 + case <-wait: 336 336 + } 337 337 + 338 338 + e.l.Info("waited for container", "name", containerID) 339 339 + 340 340 + info, err := e.docker.ContainerInspect(ctx, containerID) 341 341 + if err != nil { 342 342 + return nil, err 343 343 + } 344 344 + 345 345 + return info.State, nil 346 346 + } 347 347 + 348 348 + func (e *Engine) tailStep(ctx context.Context, wfLogger *models.WorkflowLogger, containerID string, wid models.WorkflowId, stepIdx int, step models.Step) error { 349 349 + if wfLogger == nil { 350 350 + return nil 351 351 + } 352 352 + 353 353 + logs, err := e.docker.ContainerLogs(ctx, containerID, container.LogsOptions{ 354 354 + Follow: true, 355 355 + ShowStdout: true, 356 356 + ShowStderr: true, 357 357 + Details: false, 358 358 + Timestamps: false, 359 359 + }) 360 360 + if err != nil { 361 361 + return err 362 362 + } 363 363 + 364 364 + _, err = stdcopy.StdCopy( 365 365 + wfLogger.DataWriter("stdout"), 366 366 + wfLogger.DataWriter("stderr"), 367 367 + logs, 368 368 + ) 369 369 + if err != nil && err != io.EOF && !errors.Is(err, context.DeadlineExceeded) { 370 370 + return fmt.Errorf("failed to copy logs: %w", err) 371 371 + } 372 372 + 373 373 + return nil 374 374 + } 375 375 + 376 376 + func (e *Engine) DestroyStep(ctx context.Context, containerID string) error { 377 377 + err := e.docker.ContainerKill(ctx, containerID, "9") // SIGKILL 378 378 + if err != nil && !isErrContainerNotFoundOrNotRunning(err) { 379 379 + return err 380 380 + } 381 381 + 382 382 + if err := e.docker.ContainerRemove(ctx, containerID, container.RemoveOptions{ 383 383 + RemoveVolumes: true, 384 384 + RemoveLinks: false, 385 385 + Force: false, 386 386 + }); err != nil && !isErrContainerNotFoundOrNotRunning(err) { 387 387 + return err 388 388 + } 389 389 + 390 390 + return nil 391 391 + } 392 392 + 393 393 + func (e *Engine) DestroyWorkflow(ctx context.Context, wid models.WorkflowId) error { 394 394 + e.cleanupMu.Lock() 395 395 + key := wid.String() 396 396 + 397 397 + fns := e.cleanup[key] 398 398 + delete(e.cleanup, key) 399 399 + e.cleanupMu.Unlock() 400 400 + 401 401 + for _, fn := range fns { 402 402 + if err := fn(ctx); err != nil { 403 403 + e.l.Error("failed to cleanup workflow resource", "workflowId", wid, "error", err) 404 404 + } 405 405 + } 406 406 + return nil 407 407 + } 408 408 + 409 409 + func (e *Engine) registerCleanup(wid models.WorkflowId, fn cleanupFunc) { 410 410 + e.cleanupMu.Lock() 411 411 + defer e.cleanupMu.Unlock() 412 412 + 413 413 + key := wid.String() 414 414 + e.cleanup[key] = append(e.cleanup[key], fn) 415 415 + } 416 416 + 417 417 + func workspaceVolume(wid models.WorkflowId) string { 418 418 + return fmt.Sprintf("workspace-%s", wid) 419 419 + } 420 420 + 421 421 + func nixVolume(wid models.WorkflowId) string { 422 422 + return fmt.Sprintf("nix-%s", wid) 423 423 + } 424 424 + 425 425 + func networkName(wid models.WorkflowId) string { 426 426 + return fmt.Sprintf("workflow-network-%s", wid) 427 427 + } 428 428 + 429 429 + func hostConfig(wid models.WorkflowId) *container.HostConfig { 430 430 + hostConfig := &container.HostConfig{ 431 431 + Mounts: []mount.Mount{ 432 432 + { 433 433 + Type: mount.TypeVolume, 434 434 + Source: workspaceVolume(wid), 435 435 + Target: workspaceDir, 436 436 + }, 437 437 + { 438 438 + Type: mount.TypeVolume, 439 439 + Source: nixVolume(wid), 440 440 + Target: "/nix", 441 441 + }, 442 442 + { 443 443 + Type: mount.TypeTmpfs, 444 444 + Target: "/tmp", 445 445 + ReadOnly: false, 446 446 + TmpfsOptions: &mount.TmpfsOptions{ 447 447 + Mode: 0o1777, // world-writeable sticky bit 448 448 + Options: [][]string{ 449 449 + {"exec"}, 450 450 + }, 451 451 + }, 452 452 + }, 453 453 + { 454 454 + Type: mount.TypeVolume, 455 455 + Source: "etc-nix-" + wid.String(), 456 456 + Target: "/etc/nix", 457 457 + }, 458 458 + }, 459 459 + ReadonlyRootfs: false, 460 460 + CapDrop: []string{"ALL"}, 461 461 + CapAdd: []string{"CAP_DAC_OVERRIDE"}, 462 462 + SecurityOpt: []string{"no-new-privileges"}, 463 463 + ExtraHosts: []string{"host.docker.internal:host-gateway"}, 464 464 + } 465 465 + 466 466 + return hostConfig 467 467 + } 468 468 + 469 469 + // thanks woodpecker 470 470 + func isErrContainerNotFoundOrNotRunning(err error) bool { 471 471 + // Error response from daemon: Cannot kill container: ...: No such container: ... 472 472 + // Error response from daemon: Cannot kill container: ...: Container ... is not running" 473 473 + // Error response from podman daemon: can only kill running containers. ... is in state exited 474 474 + // Error: No such container: ... 475 475 + return err != nil && (strings.Contains(err.Error(), "No such container") || strings.Contains(err.Error(), "is not running") || strings.Contains(err.Error(), "can only kill running containers")) 476 476 + }

+7

spindle/engines/nixery/errors.go

··· 1 1 + package nixery 2 2 + 3 3 + import "errors" 4 4 + 5 5 + var ( 6 6 + ErrOOMKilled = errors.New("oom killed") 7 7 + )

+17

spindle/models/engine.go

··· 1 1 + package models 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "time" 6 6 + 7 7 + "tangled.sh/tangled.sh/core/api/tangled" 8 8 + "tangled.sh/tangled.sh/core/spindle/secrets" 9 9 + ) 10 10 + 11 11 + type Engine interface { 12 12 + InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipeline) (*Workflow, error) 13 13 + SetupWorkflow(ctx context.Context, wid WorkflowId, wf *Workflow) error 14 14 + WorkflowTimeout() time.Duration 15 15 + DestroyWorkflow(ctx context.Context, wid WorkflowId) error 16 16 + RunStep(ctx context.Context, wid WorkflowId, w *Workflow, idx int, secrets []secrets.UnlockedSecret, wfLogger *WorkflowLogger) error 17 17 + }

+3 -3

spindle/models/models.go

··· 104 104 func NewControlLogLine(idx int, step Step) LogLine { 105 105 return LogLine{ 106 106 Kind: LogKindControl, 107 107 - Content: step.Name, 107 107 + Content: step.Name(), 108 108 StepId: idx, 109 109 - StepKind: step.Kind, 110 110 - StepCommand: step.Command, 109 109 + StepKind: step.Kind(), 110 110 + StepCommand: step.Command(), 111 111 } 112 112 }

+8 -103

spindle/models/pipeline.go

··· 1 1 package models 2 2 3 3 - import ( 4 4 - "path" 5 5 - 6 6 - "tangled.sh/tangled.sh/core/api/tangled" 7 7 - "tangled.sh/tangled.sh/core/spindle/config" 8 8 - ) 9 9 - 10 3 type Pipeline struct { 11 4 RepoOwner string 12 5 RepoName string 13 13 - Workflows []Workflow 6 6 + Workflows map[Engine][]Workflow 14 7 } 15 8 16 16 - type Step struct { 17 17 - Command string 18 18 - Name string 19 19 - Environment map[string]string 20 20 - Kind StepKind 9 9 + type Step interface { 10 10 + Name() string 11 11 + Command() string 12 12 + Kind() StepKind 21 13 } 22 14 23 15 type StepKind int ··· 30 22 ) 31 23 32 24 type Workflow struct { 33 33 - Steps []Step 34 34 - Environment map[string]string 35 35 - Name string 36 36 - Image string 37 37 - } 38 38 - 39 39 - // setupSteps get added to start of Steps 40 40 - type setupSteps []Step 41 41 - 42 42 - // addStep adds a step to the beginning of the workflow's steps. 43 43 - func (ss *setupSteps) addStep(step Step) { 44 44 - *ss = append(*ss, step) 45 45 - } 46 46 - 47 47 - // ToPipeline converts a tangled.Pipeline into a model.Pipeline. 48 48 - // In the process, dependencies are resolved: nixpkgs deps 49 49 - // are constructed atop nixery and set as the Workflow.Image, 50 50 - // and ones from custom registries 51 51 - func ToPipeline(pl tangled.Pipeline, cfg config.Config) *Pipeline { 52 52 - workflows := []Workflow{} 53 53 - 54 54 - for _, twf := range pl.Workflows { 55 55 - swf := &Workflow{} 56 56 - for _, tstep := range twf.Steps { 57 57 - sstep := Step{} 58 58 - sstep.Environment = stepEnvToMap(tstep.Environment) 59 59 - sstep.Command = tstep.Command 60 60 - sstep.Name = tstep.Name 61 61 - sstep.Kind = StepKindUser 62 62 - swf.Steps = append(swf.Steps, sstep) 63 63 - } 64 64 - swf.Name = twf.Name 65 65 - swf.Environment = workflowEnvToMap(twf.Environment) 66 66 - swf.Image = workflowImage(twf.Dependencies, cfg.Pipelines.Nixery) 67 67 - 68 68 - setup := &setupSteps{} 69 69 - 70 70 - setup.addStep(nixConfStep()) 71 71 - setup.addStep(cloneStep(*twf, *pl.TriggerMetadata, cfg.Server.Dev)) 72 72 - // this step could be empty 73 73 - if s := dependencyStep(*twf); s != nil { 74 74 - setup.addStep(*s) 75 75 - } 76 76 - 77 77 - // append setup steps in order to the start of workflow steps 78 78 - swf.Steps = append(*setup, swf.Steps...) 79 79 - 80 80 - workflows = append(workflows, *swf) 81 81 - } 82 82 - repoOwner := pl.TriggerMetadata.Repo.Did 83 83 - repoName := pl.TriggerMetadata.Repo.Repo 84 84 - return &Pipeline{ 85 85 - RepoOwner: repoOwner, 86 86 - RepoName: repoName, 87 87 - Workflows: workflows, 88 88 - } 89 89 - } 90 90 - 91 91 - func workflowEnvToMap(envs []*tangled.Pipeline_Pair) map[string]string { 92 92 - envMap := map[string]string{} 93 93 - for _, env := range envs { 94 94 - if env != nil { 95 95 - envMap[env.Key] = env.Value 96 96 - } 97 97 - } 98 98 - return envMap 99 99 - } 100 100 - 101 101 - func stepEnvToMap(envs []*tangled.Pipeline_Pair) map[string]string { 102 102 - envMap := map[string]string{} 103 103 - for _, env := range envs { 104 104 - if env != nil { 105 105 - envMap[env.Key] = env.Value 106 106 - } 107 107 - } 108 108 - return envMap 109 109 - } 110 110 - 111 111 - func workflowImage(deps []*tangled.Pipeline_Dependency, nixery string) string { 112 112 - var dependencies string 113 113 - for _, d := range deps { 114 114 - if d.Registry == "nixpkgs" { 115 115 - dependencies = path.Join(d.Packages...) 116 116 - } 117 117 - } 118 118 - 119 119 - // load defaults from somewhere else 120 120 - dependencies = path.Join(dependencies, "bash", "git", "coreutils", "nix") 121 121 - 122 122 - return path.Join(nixery, dependencies) 25 25 + Steps []Step 26 26 + Name string 27 27 + Data any 123 28 }

+10 -13

spindle/models/setup_steps.go spindle/engines/nixery/setup_steps.go

··· 1 1 - package models 1 1 + package nixery 2 2 3 3 import ( 4 4 "fmt" ··· 13 13 setupCmd := `echo 'extra-experimental-features = nix-command flakes' >> /etc/nix/nix.conf 14 14 echo 'build-users-group = ' >> /etc/nix/nix.conf` 15 15 return Step{ 16 16 - Command: setupCmd, 17 17 - Name: "Configure Nix", 16 16 + command: setupCmd, 17 17 + name: "Configure Nix", 18 18 } 19 19 } 20 20 ··· 81 81 commands = append(commands, "git checkout FETCH_HEAD") 82 82 83 83 cloneStep := Step{ 84 84 - Command: strings.Join(commands, "\n"), 85 85 - Name: "Clone repository into workspace", 84 84 + command: strings.Join(commands, "\n"), 85 85 + name: "Clone repository into workspace", 86 86 } 87 87 return cloneStep 88 88 } ··· 91 91 // For dependencies using a custom registry (i.e. not nixpkgs), it collects 92 92 // all packages and adds a single 'nix profile install' step to the 93 93 // beginning of the workflow's step list. 94 94 - func dependencyStep(twf tangled.Pipeline_Workflow) *Step { 94 94 + func dependencyStep(deps map[string][]string) *Step { 95 95 var customPackages []string 96 96 97 97 - for _, d := range twf.Dependencies { 98 98 - registry := d.Registry 99 99 - packages := d.Packages 100 100 - 97 97 + for registry, packages := range deps { 101 98 if registry == "nixpkgs" { 102 99 continue 103 100 } ··· 115 112 installCmd := "nix --extra-experimental-features nix-command --extra-experimental-features flakes profile install" 116 113 cmd := fmt.Sprintf("%s %s", installCmd, strings.Join(customPackages, " ")) 117 114 installStep := Step{ 118 118 - Command: cmd, 119 119 - Name: "Install custom dependencies", 120 120 - Environment: map[string]string{ 115 115 + command: cmd, 116 116 + name: "Install custom dependencies", 117 117 + environment: map[string]string{ 121 118 "NIX_NO_COLOR": "1", 122 119 "NIX_SHOW_DOWNLOAD_PROGRESS": "0", 123 120 },

+38 -8

spindle/server.go

··· 20 20 "tangled.sh/tangled.sh/core/spindle/config" 21 21 "tangled.sh/tangled.sh/core/spindle/db" 22 22 "tangled.sh/tangled.sh/core/spindle/engine" 23 23 + "tangled.sh/tangled.sh/core/spindle/engines/nixery" 23 24 "tangled.sh/tangled.sh/core/spindle/models" 24 25 "tangled.sh/tangled.sh/core/spindle/queue" 25 26 "tangled.sh/tangled.sh/core/spindle/secrets" ··· 39 40 e *rbac.Enforcer 40 41 l *slog.Logger 41 42 n *notifier.Notifier 42 42 - eng *engine.Engine 43 43 + engs map[string]models.Engine 43 44 jq *queue.Queue 44 45 cfg *config.Config 45 46 ks *eventconsumer.Consumer ··· 93 94 return fmt.Errorf("unknown secrets provider: %s", cfg.Server.Secrets.Provider) 94 95 } 95 96 96 96 - eng, err := engine.New(ctx, cfg, d, &n, vault) 97 97 + nixeryEng, err := nixery.New(ctx, cfg) 97 98 if err != nil { 98 99 return err 99 100 } ··· 128 129 db: d, 129 130 l: logger, 130 131 n: &n, 131 131 - eng: eng, 132 132 + engs: map[string]models.Engine{"nixery": nixeryEng}, 132 133 jq: jq, 133 134 cfg: cfg, 134 135 res: resolver, ··· 216 217 Logger: logger, 217 218 Db: s.db, 218 219 Enforcer: s.e, 219 219 - Engine: s.eng, 220 220 + Engines: s.engs, 220 221 Config: s.cfg, 221 222 Resolver: s.res, 222 223 Vault: s.vault, ··· 261 262 Rkey: msg.Rkey, 262 263 } 263 264 265 265 + workflows := make(map[models.Engine][]models.Workflow) 266 266 + 264 267 for _, w := range tpl.Workflows { 265 268 if w != nil { 266 266 - err := s.db.StatusPending(models.WorkflowId{ 269 269 + if _, ok := s.engs[w.Engine]; !ok { 270 270 + err = s.db.StatusFailed(models.WorkflowId{ 271 271 + PipelineId: pipelineId, 272 272 + Name: w.Name, 273 273 + }, fmt.Sprintf("unknown engine %#v", w.Engine), -1, s.n) 274 274 + if err != nil { 275 275 + return err 276 276 + } 277 277 + 278 278 + continue 279 279 + } 280 280 + 281 281 + eng := s.engs[w.Engine] 282 282 + 283 283 + if _, ok := workflows[eng]; !ok { 284 284 + workflows[eng] = []models.Workflow{} 285 285 + } 286 286 + 287 287 + ewf, err := s.engs[w.Engine].InitWorkflow(*w, tpl) 288 288 + if err != nil { 289 289 + return err 290 290 + } 291 291 + 292 292 + workflows[eng] = append(workflows[eng], *ewf) 293 293 + 294 294 + err = s.db.StatusPending(models.WorkflowId{ 267 295 PipelineId: pipelineId, 268 296 Name: w.Name, 269 297 }, s.n) ··· 273 301 } 274 302 } 275 303 276 276 - spl := models.ToPipeline(tpl, *s.cfg) 277 277 - 278 304 ok := s.jq.Enqueue(queue.Job{ 279 305 Run: func() error { 280 280 - s.eng.StartWorkflows(ctx, spl, pipelineId) 306 306 + engine.StartWorkflows(s.l, s.vault, s.cfg, s.db, s.n, ctx, &models.Pipeline{ 307 307 + RepoOwner: tpl.TriggerMetadata.Repo.Did, 308 308 + RepoName: tpl.TriggerMetadata.Repo.Repo, 309 309 + Workflows: workflows, 310 310 + }, pipelineId) 281 311 return nil 282 312 }, 283 313 OnFail: func(jobError error) {

+1 -2

spindle/stream.go

··· 9 9 "strconv" 10 10 "time" 11 11 12 12 - "tangled.sh/tangled.sh/core/spindle/engine" 13 12 "tangled.sh/tangled.sh/core/spindle/models" 14 13 15 14 "github.com/go-chi/chi/v5" ··· 143 142 } 144 143 isFinished := models.StatusKind(status.Status).IsFinish() 145 144 146 146 - filePath := engine.LogFilePath(s.cfg.Pipelines.LogDir, wid) 145 145 + filePath := models.LogFilePath(s.cfg.Server.LogDir, wid) 147 146 148 147 config := tail.Config{ 149 148 Follow: !isFinished,

+2 -2

spindle/xrpc/xrpc.go

··· 17 17 "tangled.sh/tangled.sh/core/rbac" 18 18 "tangled.sh/tangled.sh/core/spindle/config" 19 19 "tangled.sh/tangled.sh/core/spindle/db" 20 20 - "tangled.sh/tangled.sh/core/spindle/engine" 20 20 + "tangled.sh/tangled.sh/core/spindle/models" 21 21 "tangled.sh/tangled.sh/core/spindle/secrets" 22 22 ) 23 23 ··· 27 27 Logger *slog.Logger 28 28 Db *db.DB 29 29 Enforcer *rbac.Enforcer 30 30 - Engine *engine.Engine 30 30 + Engines map[string]models.Engine 31 31 Config *config.Config 32 32 Resolver *idresolver.Resolver 33 33 Vault secrets.Manager

+17 -36

workflow/compile.go

··· 1 1 package workflow 2 2 3 3 import ( 4 4 + "errors" 4 5 "fmt" 5 6 6 7 "tangled.sh/tangled.sh/core/api/tangled" ··· 63 64 return fmt.Sprintf("warning: %s: %s: %s", w.Path, w.Type, w.Reason) 64 65 } 65 66 67 67 + var ( 68 68 + MissingEngine error = errors.New("missing engine") 69 69 + ) 70 70 + 66 71 type WarningKind string 67 72 68 73 var ( ··· 95 100 for _, wf := range p { 96 101 cw := compiler.compileWorkflow(wf) 97 102 98 98 - // empty workflows are not added to the pipeline 99 99 - if len(cw.Steps) == 0 { 103 103 + if cw == nil { 100 104 continue 101 105 } 102 106 103 103 - cp.Workflows = append(cp.Workflows, &cw) 107 107 + cp.Workflows = append(cp.Workflows, cw) 104 108 } 105 109 106 110 return cp 107 111 } 108 112 109 109 - func (compiler *Compiler) compileWorkflow(w Workflow) tangled.Pipeline_Workflow { 110 110 - cw := tangled.Pipeline_Workflow{} 113 113 + func (compiler *Compiler) compileWorkflow(w Workflow) *tangled.Pipeline_Workflow { 114 114 + cw := &tangled.Pipeline_Workflow{} 111 115 112 116 if !w.Match(compiler.Trigger) { 113 117 compiler.Diagnostics.AddWarning( ··· 115 119 WorkflowSkipped, 116 120 fmt.Sprintf("did not match trigger %s", compiler.Trigger.Kind), 117 121 ) 118 118 - return cw 119 119 - } 120 120 - 121 121 - if len(w.Steps) == 0 { 122 122 - compiler.Diagnostics.AddWarning( 123 123 - w.Name, 124 124 - WorkflowSkipped, 125 125 - "empty workflow", 126 126 - ) 127 127 - return cw 122 122 + return nil 128 123 } 129 124 130 125 // validate clone options 131 126 compiler.analyzeCloneOptions(w) 132 127 133 128 cw.Name = w.Name 134 134 - cw.Dependencies = w.Dependencies.AsRecord() 135 135 - for _, s := range w.Steps { 136 136 - step := tangled.Pipeline_Step{ 137 137 - Command: s.Command, 138 138 - Name: s.Name, 139 139 - } 140 140 - for k, v := range s.Environment { 141 141 - e := &tangled.Pipeline_Pair{ 142 142 - Key: k, 143 143 - Value: v, 144 144 - } 145 145 - step.Environment = append(step.Environment, e) 146 146 - } 147 147 - cw.Steps = append(cw.Steps, &step) 129 129 + 130 130 + if w.Engine == "" { 131 131 + compiler.Diagnostics.AddError(w.Name, MissingEngine) 132 132 + return nil 148 133 } 149 149 - for k, v := range w.Environment { 150 150 - e := &tangled.Pipeline_Pair{ 151 151 - Key: k, 152 152 - Value: v, 153 153 - } 154 154 - cw.Environment = append(cw.Environment, e) 155 155 - } 134 134 + 135 135 + cw.Engine = w.Engine 136 136 + cw.Raw = w.Raw 156 137 157 138 o := w.CloneOpts.AsRecord() 158 139 cw.Clone = &o

+23 -29

workflow/compile_test.go

··· 26 26 27 27 func TestCompileWorkflow_MatchingWorkflowWithSteps(t *testing.T) { 28 28 wf := Workflow{ 29 29 - Name: ".tangled/workflows/test.yml", 30 30 - When: when, 31 31 - Steps: []Step{ 32 32 - {Name: "Test", Command: "go test ./..."}, 33 33 - }, 29 29 + Name: ".tangled/workflows/test.yml", 30 30 + Engine: "nixery", 31 31 + When: when, 34 32 CloneOpts: CloneOpts{}, // default true 35 33 } 36 34 ··· 43 41 assert.False(t, c.Diagnostics.IsErr()) 44 42 } 45 43 46 46 - func TestCompileWorkflow_EmptySteps(t *testing.T) { 47 47 - wf := Workflow{ 48 48 - Name: ".tangled/workflows/empty.yml", 49 49 - When: when, 50 50 - Steps: []Step{}, // no steps 51 51 - } 52 52 - 53 53 - c := Compiler{Trigger: trigger} 54 54 - cp := c.Compile([]Workflow{wf}) 55 55 - 56 56 - assert.Len(t, cp.Workflows, 0) 57 57 - assert.Len(t, c.Diagnostics.Warnings, 1) 58 58 - assert.Equal(t, WorkflowSkipped, c.Diagnostics.Warnings[0].Type) 59 59 - } 60 60 - 61 44 func TestCompileWorkflow_TriggerMismatch(t *testing.T) { 62 45 wf := Workflow{ 63 63 - Name: ".tangled/workflows/mismatch.yml", 46 46 + Name: ".tangled/workflows/mismatch.yml", 47 47 + Engine: "nixery", 64 48 When: []Constraint{ 65 49 { 66 50 Event: []string{"push"}, 67 51 Branch: []string{"master"}, // different branch 68 52 }, 69 53 }, 70 70 - Steps: []Step{ 71 71 - {Name: "Lint", Command: "golint ./..."}, 72 72 - }, 73 54 } 74 55 75 56 c := Compiler{Trigger: trigger} ··· 82 63 83 64 func TestCompileWorkflow_CloneFalseWithShallowTrue(t *testing.T) { 84 65 wf := Workflow{ 85 85 - Name: ".tangled/workflows/clone_skip.yml", 86 86 - When: when, 87 87 - Steps: []Step{ 88 88 - {Name: "Skip", Command: "echo skip"}, 89 89 - }, 66 66 + Name: ".tangled/workflows/clone_skip.yml", 67 67 + Engine: "nixery", 68 68 + When: when, 90 69 CloneOpts: CloneOpts{ 91 70 Skip: true, 92 71 Depth: 1, ··· 101 80 assert.Len(t, c.Diagnostics.Warnings, 1) 102 81 assert.Equal(t, InvalidConfiguration, c.Diagnostics.Warnings[0].Type) 103 82 } 83 83 + 84 84 + func TestCompileWorkflow_MissingEngine(t *testing.T) { 85 85 + wf := Workflow{ 86 86 + Name: ".tangled/workflows/missing_engine.yml", 87 87 + When: when, 88 88 + Engine: "", 89 89 + } 90 90 + 91 91 + c := Compiler{Trigger: trigger} 92 92 + cp := c.Compile([]Workflow{wf}) 93 93 + 94 94 + assert.Len(t, cp.Workflows, 0) 95 95 + assert.Len(t, c.Diagnostics.Errors, 1) 96 96 + assert.Equal(t, MissingEngine, c.Diagnostics.Errors[0].Error) 97 97 + }

+6 -33

workflow/def.go

··· 24 24 25 25 // this is simply a structural representation of the workflow file 26 26 Workflow struct { 27 27 - Name string `yaml:"-"` // name of the workflow file 28 28 - When []Constraint `yaml:"when"` 29 29 - Dependencies Dependencies `yaml:"dependencies"` 30 30 - Steps []Step `yaml:"steps"` 31 31 - Environment map[string]string `yaml:"environment"` 32 32 - CloneOpts CloneOpts `yaml:"clone"` 27 27 + Name string `yaml:"-"` // name of the workflow file 28 28 + Engine string `yaml:"engine"` 29 29 + When []Constraint `yaml:"when"` 30 30 + CloneOpts CloneOpts `yaml:"clone"` 31 31 + Raw string `yaml:"-"` 33 32 } 34 33 35 34 Constraint struct { 36 35 Event StringList `yaml:"event"` 37 36 Branch StringList `yaml:"branch"` // this is optional, and only applied on "push" events 38 37 } 39 39 - 40 40 - Dependencies map[string][]string 41 38 42 39 CloneOpts struct { 43 40 Skip bool `yaml:"skip"` 44 41 Depth int `yaml:"depth"` 45 42 IncludeSubmodules bool `yaml:"submodules"` 46 46 - } 47 47 - 48 48 - Step struct { 49 49 - Name string `yaml:"name"` 50 50 - Command string `yaml:"command"` 51 51 - Environment map[string]string `yaml:"environment"` 52 43 } 53 44 54 45 StringList []string ··· 77 68 } 78 69 79 70 wf.Name = name 71 71 + wf.Raw = string(contents) 80 72 81 73 return wf, nil 82 74 } ··· 173 165 } 174 166 175 167 return errors.New("failed to unmarshal StringOrSlice") 176 176 - } 177 177 - 178 178 - // conversion utilities to atproto records 179 179 - func (d Dependencies) AsRecord() []*tangled.Pipeline_Dependency { 180 180 - var deps []*tangled.Pipeline_Dependency 181 181 - for registry, packages := range d { 182 182 - deps = append(deps, &tangled.Pipeline_Dependency{ 183 183 - Registry: registry, 184 184 - Packages: packages, 185 185 - }) 186 186 - } 187 187 - return deps 188 188 - } 189 189 - 190 190 - func (s Step) AsRecord() tangled.Pipeline_Step { 191 191 - return tangled.Pipeline_Step{ 192 192 - Command: s.Command, 193 193 - Name: s.Name, 194 194 - } 195 168 } 196 169 197 170 func (c CloneOpts) AsRecord() tangled.Pipeline_CloneOpts {

+1 -86

workflow/def_test.go

··· 10 10 yamlData := ` 11 11 when: 12 12 - event: ["push", "pull_request"] 13 13 - branch: ["main", "develop"] 14 14 - 15 15 - dependencies: 16 16 - nixpkgs: 17 17 - - go 18 18 - - git 19 19 - - curl 20 20 - 21 21 - steps: 22 22 - - name: "Test" 23 23 - command: | 24 24 - go test ./...` 13 13 + branch: ["main", "develop"]` 25 14 26 15 wf, err := FromFile("test.yml", []byte(yamlData)) 27 16 assert.NoError(t, err, "YAML should unmarshal without error") ··· 30 19 assert.ElementsMatch(t, []string{"main", "develop"}, wf.When[0].Branch) 31 20 assert.ElementsMatch(t, []string{"push", "pull_request"}, wf.When[0].Event) 32 21 33 33 - assert.Len(t, wf.Steps, 1) 34 34 - assert.Equal(t, "Test", wf.Steps[0].Name) 35 35 - assert.Equal(t, "go test ./...", wf.Steps[0].Command) 36 36 - 37 37 - pkgs, ok := wf.Dependencies["nixpkgs"] 38 38 - assert.True(t, ok, "`nixpkgs` should be present in dependencies") 39 39 - assert.ElementsMatch(t, []string{"go", "git", "curl"}, pkgs) 40 40 - 41 22 assert.False(t, wf.CloneOpts.Skip, "Skip should default to false") 42 23 } 43 24 44 44 - func TestUnmarshalCustomRegistry(t *testing.T) { 45 45 - yamlData := ` 46 46 - when: 47 47 - - event: push 48 48 - branch: main 49 49 - 50 50 - dependencies: 51 51 - git+https://tangled.sh/@oppi.li/tbsp: 52 52 - - tbsp 53 53 - git+https://git.peppe.rs/languages/statix: 54 54 - - statix 55 55 - 56 56 - steps: 57 57 - - name: "Check" 58 58 - command: | 59 59 - statix check` 60 60 - 61 61 - wf, err := FromFile("test.yml", []byte(yamlData)) 62 62 - assert.NoError(t, err, "YAML should unmarshal without error") 63 63 - 64 64 - assert.ElementsMatch(t, []string{"push"}, wf.When[0].Event) 65 65 - assert.ElementsMatch(t, []string{"main"}, wf.When[0].Branch) 66 66 - 67 67 - assert.ElementsMatch(t, []string{"tbsp"}, wf.Dependencies["git+https://tangled.sh/@oppi.li/tbsp"]) 68 68 - assert.ElementsMatch(t, []string{"statix"}, wf.Dependencies["git+https://git.peppe.rs/languages/statix"]) 69 69 - } 70 70 - 71 25 func TestUnmarshalCloneFalse(t *testing.T) { 72 26 yamlData := ` 73 27 when: ··· 75 29 76 30 clone: 77 31 skip: true 78 78 - 79 79 - dependencies: 80 80 - nixpkgs: 81 81 - - python3 82 82 - 83 83 - steps: 84 84 - - name: Notify 85 85 - command: | 86 86 - python3 ./notify.py 87 32 ` 88 33 89 34 wf, err := FromFile("test.yml", []byte(yamlData)) ··· 93 38 94 39 assert.True(t, wf.CloneOpts.Skip, "Skip should be false") 95 40 } 96 96 - 97 97 - func TestUnmarshalEnv(t *testing.T) { 98 98 - yamlData := ` 99 99 - when: 100 100 - - event: ["pull_request_close"] 101 101 - 102 102 - clone: 103 103 - skip: false 104 104 - 105 105 - environment: 106 106 - HOME: /home/foo bar/baz 107 107 - CGO_ENABLED: 1 108 108 - 109 109 - steps: 110 110 - - name: Something 111 111 - command: echo "hello" 112 112 - environment: 113 113 - FOO: bar 114 114 - BAZ: qux 115 115 - ` 116 116 - 117 117 - wf, err := FromFile("test.yml", []byte(yamlData)) 118 118 - assert.NoError(t, err) 119 119 - 120 120 - assert.Len(t, wf.Environment, 2) 121 121 - assert.Equal(t, "/home/foo bar/baz", wf.Environment["HOME"]) 122 122 - assert.Equal(t, "1", wf.Environment["CGO_ENABLED"]) 123 123 - assert.Equal(t, "bar", wf.Steps[0].Environment["FOO"]) 124 124 - assert.Equal(t, "qux", wf.Steps[0].Environment["BAZ"]) 125 125 - }