+53 -600

api/tangled/cbor_gen.go

··· 2728 2729 return nil 2730 } 2731 - func (t *Pipeline_Dependency) MarshalCBOR(w io.Writer) error { 2732 - if t == nil { 2733 - _, err := w.Write(cbg.CborNull) 2734 - return err 2735 - } 2736 - 2737 - cw := cbg.NewCborWriter(w) 2738 - 2739 - if _, err := cw.Write([]byte{162}); err != nil { 2740 - return err 2741 - } 2742 - 2743 - // t.Packages ([]string) (slice) 2744 - if len("packages") > 1000000 { 2745 - return xerrors.Errorf("Value in field \"packages\" was too long") 2746 - } 2747 - 2748 - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("packages"))); err != nil { 2749 - return err 2750 - } 2751 - if _, err := cw.WriteString(string("packages")); err != nil { 2752 - return err 2753 - } 2754 - 2755 - if len(t.Packages) > 8192 { 2756 - return xerrors.Errorf("Slice value in field t.Packages was too long") 2757 - } 2758 - 2759 - if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Packages))); err != nil { 2760 - return err 2761 - } 2762 - for _, v := range t.Packages { 2763 - if len(v) > 1000000 { 2764 - return xerrors.Errorf("Value in field v was too long") 2765 - } 2766 - 2767 - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(v))); err != nil { 2768 - return err 2769 - } 2770 - if _, err := cw.WriteString(string(v)); err != nil { 2771 - return err 2772 - } 2773 - 2774 - } 2775 - 2776 - // t.Registry (string) (string) 2777 - if len("registry") > 1000000 { 2778 - return xerrors.Errorf("Value in field \"registry\" was too long") 2779 - } 2780 - 2781 - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("registry"))); err != nil { 2782 - return err 2783 - } 2784 - if _, err := cw.WriteString(string("registry")); err != nil { 2785 - return err 2786 - } 2787 - 2788 - if len(t.Registry) > 1000000 { 2789 - return xerrors.Errorf("Value in field t.Registry was too long") 2790 - } 2791 - 2792 - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Registry))); err != nil { 2793 - return err 2794 - } 2795 - if _, err := cw.WriteString(string(t.Registry)); err != nil { 2796 - return err 2797 - } 2798 - return nil 2799 - } 2800 - 2801 - func (t *Pipeline_Dependency) UnmarshalCBOR(r io.Reader) (err error) { 2802 - *t = Pipeline_Dependency{} 2803 - 2804 - cr := cbg.NewCborReader(r) 2805 - 2806 - maj, extra, err := cr.ReadHeader() 2807 - if err != nil { 2808 - return err 2809 - } 2810 - defer func() { 2811 - if err == io.EOF { 2812 - err = io.ErrUnexpectedEOF 2813 - } 2814 - }() 2815 - 2816 - if maj != cbg.MajMap { 2817 - return fmt.Errorf("cbor input should be of type map") 2818 - } 2819 - 2820 - if extra > cbg.MaxLength { 2821 - return fmt.Errorf("Pipeline_Dependency: map struct too large (%d)", extra) 2822 - } 2823 - 2824 - n := extra 2825 - 2826 - nameBuf := make([]byte, 8) 2827 - for i := uint64(0); i < n; i++ { 2828 - nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) 2829 - if err != nil { 2830 - return err 2831 - } 2832 - 2833 - if !ok { 2834 - // Field doesn't exist on this type, so ignore it 2835 - if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { 2836 - return err 2837 - } 2838 - continue 2839 - } 2840 - 2841 - switch string(nameBuf[:nameLen]) { 2842 - // t.Packages ([]string) (slice) 2843 - case "packages": 2844 - 2845 - maj, extra, err = cr.ReadHeader() 2846 - if err != nil { 2847 - return err 2848 - } 2849 - 2850 - if extra > 8192 { 2851 - return fmt.Errorf("t.Packages: array too large (%d)", extra) 2852 - } 2853 - 2854 - if maj != cbg.MajArray { 2855 - return fmt.Errorf("expected cbor array") 2856 - } 2857 - 2858 - if extra > 0 { 2859 - t.Packages = make([]string, extra) 2860 - } 2861 - 2862 - for i := 0; i < int(extra); i++ { 2863 - { 2864 - var maj byte 2865 - var extra uint64 2866 - var err error 2867 - _ = maj 2868 - _ = extra 2869 - _ = err 2870 - 2871 - { 2872 - sval, err := cbg.ReadStringWithMax(cr, 1000000) 2873 - if err != nil { 2874 - return err 2875 - } 2876 - 2877 - t.Packages[i] = string(sval) 2878 - } 2879 - 2880 - } 2881 - } 2882 - // t.Registry (string) (string) 2883 - case "registry": 2884 - 2885 - { 2886 - sval, err := cbg.ReadStringWithMax(cr, 1000000) 2887 - if err != nil { 2888 - return err 2889 - } 2890 - 2891 - t.Registry = string(sval) 2892 - } 2893 - 2894 - default: 2895 - // Field doesn't exist on this type, so ignore it 2896 - if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { 2897 - return err 2898 - } 2899 - } 2900 - } 2901 - 2902 - return nil 2903 - } 2904 func (t *Pipeline_ManualTriggerData) MarshalCBOR(w io.Writer) error { 2905 if t == nil { 2906 _, err := w.Write(cbg.CborNull) ··· 3916 3917 return nil 3918 } 3919 - func (t *Pipeline_Step) MarshalCBOR(w io.Writer) error { 3920 - if t == nil { 3921 - _, err := w.Write(cbg.CborNull) 3922 - return err 3923 - } 3924 - 3925 - cw := cbg.NewCborWriter(w) 3926 - fieldCount := 3 3927 - 3928 - if t.Environment == nil { 3929 - fieldCount-- 3930 - } 3931 - 3932 - if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { 3933 - return err 3934 - } 3935 - 3936 - // t.Name (string) (string) 3937 - if len("name") > 1000000 { 3938 - return xerrors.Errorf("Value in field \"name\" was too long") 3939 - } 3940 - 3941 - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("name"))); err != nil { 3942 - return err 3943 - } 3944 - if _, err := cw.WriteString(string("name")); err != nil { 3945 - return err 3946 - } 3947 - 3948 - if len(t.Name) > 1000000 { 3949 - return xerrors.Errorf("Value in field t.Name was too long") 3950 - } 3951 - 3952 - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Name))); err != nil { 3953 - return err 3954 - } 3955 - if _, err := cw.WriteString(string(t.Name)); err != nil { 3956 - return err 3957 - } 3958 - 3959 - // t.Command (string) (string) 3960 - if len("command") > 1000000 { 3961 - return xerrors.Errorf("Value in field \"command\" was too long") 3962 - } 3963 - 3964 - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("command"))); err != nil { 3965 - return err 3966 - } 3967 - if _, err := cw.WriteString(string("command")); err != nil { 3968 - return err 3969 - } 3970 - 3971 - if len(t.Command) > 1000000 { 3972 - return xerrors.Errorf("Value in field t.Command was too long") 3973 - } 3974 - 3975 - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Command))); err != nil { 3976 - return err 3977 - } 3978 - if _, err := cw.WriteString(string(t.Command)); err != nil { 3979 - return err 3980 - } 3981 - 3982 - // t.Environment ([]*tangled.Pipeline_Pair) (slice) 3983 - if t.Environment != nil { 3984 - 3985 - if len("environment") > 1000000 { 3986 - return xerrors.Errorf("Value in field \"environment\" was too long") 3987 - } 3988 - 3989 - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("environment"))); err != nil { 3990 - return err 3991 - } 3992 - if _, err := cw.WriteString(string("environment")); err != nil { 3993 - return err 3994 - } 3995 - 3996 - if len(t.Environment) > 8192 { 3997 - return xerrors.Errorf("Slice value in field t.Environment was too long") 3998 - } 3999 - 4000 - if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Environment))); err != nil { 4001 - return err 4002 - } 4003 - for _, v := range t.Environment { 4004 - if err := v.MarshalCBOR(cw); err != nil { 4005 - return err 4006 - } 4007 - 4008 - } 4009 - } 4010 - return nil 4011 - } 4012 - 4013 - func (t *Pipeline_Step) UnmarshalCBOR(r io.Reader) (err error) { 4014 - *t = Pipeline_Step{} 4015 - 4016 - cr := cbg.NewCborReader(r) 4017 - 4018 - maj, extra, err := cr.ReadHeader() 4019 - if err != nil { 4020 - return err 4021 - } 4022 - defer func() { 4023 - if err == io.EOF { 4024 - err = io.ErrUnexpectedEOF 4025 - } 4026 - }() 4027 - 4028 - if maj != cbg.MajMap { 4029 - return fmt.Errorf("cbor input should be of type map") 4030 - } 4031 - 4032 - if extra > cbg.MaxLength { 4033 - return fmt.Errorf("Pipeline_Step: map struct too large (%d)", extra) 4034 - } 4035 - 4036 - n := extra 4037 - 4038 - nameBuf := make([]byte, 11) 4039 - for i := uint64(0); i < n; i++ { 4040 - nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) 4041 - if err != nil { 4042 - return err 4043 - } 4044 - 4045 - if !ok { 4046 - // Field doesn't exist on this type, so ignore it 4047 - if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { 4048 - return err 4049 - } 4050 - continue 4051 - } 4052 - 4053 - switch string(nameBuf[:nameLen]) { 4054 - // t.Name (string) (string) 4055 - case "name": 4056 - 4057 - { 4058 - sval, err := cbg.ReadStringWithMax(cr, 1000000) 4059 - if err != nil { 4060 - return err 4061 - } 4062 - 4063 - t.Name = string(sval) 4064 - } 4065 - // t.Command (string) (string) 4066 - case "command": 4067 - 4068 - { 4069 - sval, err := cbg.ReadStringWithMax(cr, 1000000) 4070 - if err != nil { 4071 - return err 4072 - } 4073 - 4074 - t.Command = string(sval) 4075 - } 4076 - // t.Environment ([]*tangled.Pipeline_Pair) (slice) 4077 - case "environment": 4078 - 4079 - maj, extra, err = cr.ReadHeader() 4080 - if err != nil { 4081 - return err 4082 - } 4083 - 4084 - if extra > 8192 { 4085 - return fmt.Errorf("t.Environment: array too large (%d)", extra) 4086 - } 4087 - 4088 - if maj != cbg.MajArray { 4089 - return fmt.Errorf("expected cbor array") 4090 - } 4091 - 4092 - if extra > 0 { 4093 - t.Environment = make([]*Pipeline_Pair, extra) 4094 - } 4095 - 4096 - for i := 0; i < int(extra); i++ { 4097 - { 4098 - var maj byte 4099 - var extra uint64 4100 - var err error 4101 - _ = maj 4102 - _ = extra 4103 - _ = err 4104 - 4105 - { 4106 - 4107 - b, err := cr.ReadByte() 4108 - if err != nil { 4109 - return err 4110 - } 4111 - if b != cbg.CborNull[0] { 4112 - if err := cr.UnreadByte(); err != nil { 4113 - return err 4114 - } 4115 - t.Environment[i] = new(Pipeline_Pair) 4116 - if err := t.Environment[i].UnmarshalCBOR(cr); err != nil { 4117 - return xerrors.Errorf("unmarshaling t.Environment[i] pointer: %w", err) 4118 - } 4119 - } 4120 - 4121 - } 4122 - 4123 - } 4124 - } 4125 - 4126 - default: 4127 - // Field doesn't exist on this type, so ignore it 4128 - if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { 4129 - return err 4130 - } 4131 - } 4132 - } 4133 - 4134 - return nil 4135 - } 4136 func (t *Pipeline_TriggerMetadata) MarshalCBOR(w io.Writer) error { 4137 if t == nil { 4138 _, err := w.Write(cbg.CborNull) ··· 4609 4610 cw := cbg.NewCborWriter(w) 4611 4612 - if _, err := cw.Write([]byte{165}); err != nil { 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 4613 return err 4614 } 4615 ··· 4652 return err 4653 } 4654 4655 - // t.Steps ([]*tangled.Pipeline_Step) (slice) 4656 - if len("steps") > 1000000 { 4657 - return xerrors.Errorf("Value in field \"steps\" was too long") 4658 } 4659 4660 - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("steps"))); err != nil { 4661 - return err 4662 - } 4663 - if _, err := cw.WriteString(string("steps")); err != nil { 4664 return err 4665 } 4666 - 4667 - if len(t.Steps) > 8192 { 4668 - return xerrors.Errorf("Slice value in field t.Steps was too long") 4669 - } 4670 - 4671 - if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Steps))); err != nil { 4672 - return err 4673 - } 4674 - for _, v := range t.Steps { 4675 - if err := v.MarshalCBOR(cw); err != nil { 4676 - return err 4677 - } 4678 - 4679 - } 4680 - 4681 - // t.Environment ([]*tangled.Pipeline_Pair) (slice) 4682 - if len("environment") > 1000000 { 4683 - return xerrors.Errorf("Value in field \"environment\" was too long") 4684 - } 4685 - 4686 - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("environment"))); err != nil { 4687 - return err 4688 - } 4689 - if _, err := cw.WriteString(string("environment")); err != nil { 4690 - return err 4691 - } 4692 - 4693 - if len(t.Environment) > 8192 { 4694 - return xerrors.Errorf("Slice value in field t.Environment was too long") 4695 - } 4696 - 4697 - if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Environment))); err != nil { 4698 return err 4699 } 4700 - for _, v := range t.Environment { 4701 - if err := v.MarshalCBOR(cw); err != nil { 4702 - return err 4703 - } 4704 0 0 4705 } 4706 4707 - // t.Dependencies ([]*tangled.Pipeline_Dependency) (slice) 4708 - if len("dependencies") > 1000000 { 4709 - return xerrors.Errorf("Value in field \"dependencies\" was too long") 4710 - } 4711 - 4712 - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("dependencies"))); err != nil { 4713 return err 4714 } 4715 - if _, err := cw.WriteString(string("dependencies")); err != nil { 4716 return err 4717 - } 4718 - 4719 - if len(t.Dependencies) > 8192 { 4720 - return xerrors.Errorf("Slice value in field t.Dependencies was too long") 4721 - } 4722 - 4723 - if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Dependencies))); err != nil { 4724 - return err 4725 - } 4726 - for _, v := range t.Dependencies { 4727 - if err := v.MarshalCBOR(cw); err != nil { 4728 - return err 4729 - } 4730 - 4731 } 4732 return nil 4733 } ··· 4757 4758 n := extra 4759 4760 - nameBuf := make([]byte, 12) 4761 for i := uint64(0); i < n; i++ { 4762 nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) 4763 if err != nil { ··· 4773 } 4774 4775 switch string(nameBuf[:nameLen]) { 4776 - // t.Name (string) (string) 0 0 0 0 0 0 0 0 0 0 0 4777 case "name": 4778 4779 { ··· 4804 } 4805 4806 } 4807 - // t.Steps ([]*tangled.Pipeline_Step) (slice) 4808 - case "steps": 4809 4810 - maj, extra, err = cr.ReadHeader() 4811 - if err != nil { 4812 - return err 4813 - } 4814 - 4815 - if extra > 8192 { 4816 - return fmt.Errorf("t.Steps: array too large (%d)", extra) 4817 - } 4818 - 4819 - if maj != cbg.MajArray { 4820 - return fmt.Errorf("expected cbor array") 4821 - } 4822 - 4823 - if extra > 0 { 4824 - t.Steps = make([]*Pipeline_Step, extra) 4825 - } 4826 - 4827 - for i := 0; i < int(extra); i++ { 4828 - { 4829 - var maj byte 4830 - var extra uint64 4831 - var err error 4832 - _ = maj 4833 - _ = extra 4834 - _ = err 4835 - 4836 - { 4837 - 4838 - b, err := cr.ReadByte() 4839 - if err != nil { 4840 - return err 4841 - } 4842 - if b != cbg.CborNull[0] { 4843 - if err := cr.UnreadByte(); err != nil { 4844 - return err 4845 - } 4846 - t.Steps[i] = new(Pipeline_Step) 4847 - if err := t.Steps[i].UnmarshalCBOR(cr); err != nil { 4848 - return xerrors.Errorf("unmarshaling t.Steps[i] pointer: %w", err) 4849 - } 4850 - } 4851 - 4852 - } 4853 - 4854 } 4855 - } 4856 - // t.Environment ([]*tangled.Pipeline_Pair) (slice) 4857 - case "environment": 4858 4859 - maj, extra, err = cr.ReadHeader() 4860 - if err != nil { 4861 - return err 4862 - } 4863 - 4864 - if extra > 8192 { 4865 - return fmt.Errorf("t.Environment: array too large (%d)", extra) 4866 - } 4867 - 4868 - if maj != cbg.MajArray { 4869 - return fmt.Errorf("expected cbor array") 4870 - } 4871 - 4872 - if extra > 0 { 4873 - t.Environment = make([]*Pipeline_Pair, extra) 4874 - } 4875 - 4876 - for i := 0; i < int(extra); i++ { 4877 - { 4878 - var maj byte 4879 - var extra uint64 4880 - var err error 4881 - _ = maj 4882 - _ = extra 4883 - _ = err 4884 - 4885 - { 4886 - 4887 - b, err := cr.ReadByte() 4888 - if err != nil { 4889 - return err 4890 - } 4891 - if b != cbg.CborNull[0] { 4892 - if err := cr.UnreadByte(); err != nil { 4893 - return err 4894 - } 4895 - t.Environment[i] = new(Pipeline_Pair) 4896 - if err := t.Environment[i].UnmarshalCBOR(cr); err != nil { 4897 - return xerrors.Errorf("unmarshaling t.Environment[i] pointer: %w", err) 4898 - } 4899 - } 4900 - 4901 - } 4902 - 4903 - } 4904 - } 4905 - // t.Dependencies ([]*tangled.Pipeline_Dependency) (slice) 4906 - case "dependencies": 4907 - 4908 - maj, extra, err = cr.ReadHeader() 4909 - if err != nil { 4910 - return err 4911 - } 4912 - 4913 - if extra > 8192 { 4914 - return fmt.Errorf("t.Dependencies: array too large (%d)", extra) 4915 - } 4916 - 4917 - if maj != cbg.MajArray { 4918 - return fmt.Errorf("expected cbor array") 4919 - } 4920 - 4921 - if extra > 0 { 4922 - t.Dependencies = make([]*Pipeline_Dependency, extra) 4923 - } 4924 - 4925 - for i := 0; i < int(extra); i++ { 4926 - { 4927 - var maj byte 4928 - var extra uint64 4929 - var err error 4930 - _ = maj 4931 - _ = extra 4932 - _ = err 4933 - 4934 - { 4935 - 4936 - b, err := cr.ReadByte() 4937 - if err != nil { 4938 - return err 4939 - } 4940 - if b != cbg.CborNull[0] { 4941 - if err := cr.UnreadByte(); err != nil { 4942 - return err 4943 - } 4944 - t.Dependencies[i] = new(Pipeline_Dependency) 4945 - if err := t.Dependencies[i].UnmarshalCBOR(cr); err != nil { 4946 - return xerrors.Errorf("unmarshaling t.Dependencies[i] pointer: %w", err) 4947 - } 4948 - } 4949 - 4950 - } 4951 - 4952 - } 4953 } 4954 4955 default:

··· 2728 2729 return nil 2730 } 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2731 func (t *Pipeline_ManualTriggerData) MarshalCBOR(w io.Writer) error { 2732 if t == nil { 2733 _, err := w.Write(cbg.CborNull) ··· 3743 3744 return nil 3745 } 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 3746 func (t *Pipeline_TriggerMetadata) MarshalCBOR(w io.Writer) error { 3747 if t == nil { 3748 _, err := w.Write(cbg.CborNull) ··· 4219 4220 cw := cbg.NewCborWriter(w) 4221 4222 + if _, err := cw.Write([]byte{164}); err != nil { 4223 + return err 4224 + } 4225 + 4226 + // t.Raw (string) (string) 4227 + if len("raw") > 1000000 { 4228 + return xerrors.Errorf("Value in field \"raw\" was too long") 4229 + } 4230 + 4231 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("raw"))); err != nil { 4232 + return err 4233 + } 4234 + if _, err := cw.WriteString(string("raw")); err != nil { 4235 + return err 4236 + } 4237 + 4238 + if len(t.Raw) > 1000000 { 4239 + return xerrors.Errorf("Value in field t.Raw was too long") 4240 + } 4241 + 4242 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Raw))); err != nil { 4243 + return err 4244 + } 4245 + if _, err := cw.WriteString(string(t.Raw)); err != nil { 4246 return err 4247 } 4248 ··· 4285 return err 4286 } 4287 4288 + // t.Engine (string) (string) 4289 + if len("engine") > 1000000 { 4290 + return xerrors.Errorf("Value in field \"engine\" was too long") 4291 } 4292 4293 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("engine"))); err != nil { 0 0 0 4294 return err 4295 } 4296 + if _, err := cw.WriteString(string("engine")); err != nil { 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 4297 return err 4298 } 0 0 0 0 4299 4300 + if len(t.Engine) > 1000000 { 4301 + return xerrors.Errorf("Value in field t.Engine was too long") 4302 } 4303 4304 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Engine))); err != nil { 0 0 0 0 0 4305 return err 4306 } 4307 + if _, err := cw.WriteString(string(t.Engine)); err != nil { 4308 return err 0 0 0 0 0 0 0 0 0 0 0 0 0 0 4309 } 4310 return nil 4311 } ··· 4335 4336 n := extra 4337 4338 + nameBuf := make([]byte, 6) 4339 for i := uint64(0); i < n; i++ { 4340 nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) 4341 if err != nil { ··· 4351 } 4352 4353 switch string(nameBuf[:nameLen]) { 4354 + // t.Raw (string) (string) 4355 + case "raw": 4356 + 4357 + { 4358 + sval, err := cbg.ReadStringWithMax(cr, 1000000) 4359 + if err != nil { 4360 + return err 4361 + } 4362 + 4363 + t.Raw = string(sval) 4364 + } 4365 + // t.Name (string) (string) 4366 case "name": 4367 4368 { ··· 4393 } 4394 4395 } 4396 + // t.Engine (string) (string) 4397 + case "engine": 4398 4399 + { 4400 + sval, err := cbg.ReadStringWithMax(cr, 1000000) 4401 + if err != nil { 4402 + return err 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 4403 } 0 0 0 4404 4405 + t.Engine = string(sval) 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 4406 } 4407 4408 default:

+4 -18

api/tangled/tangledpipeline.go

··· 29 Submodules bool `json:"submodules" cborgen:"submodules"` 30 } 31 32 - // Pipeline_Dependency is a "dependency" in the sh.tangled.pipeline schema. 33 - type Pipeline_Dependency struct { 34 - Packages []string `json:"packages" cborgen:"packages"` 35 - Registry string `json:"registry" cborgen:"registry"` 36 - } 37 - 38 // Pipeline_ManualTriggerData is a "manualTriggerData" in the sh.tangled.pipeline schema. 39 type Pipeline_ManualTriggerData struct { 40 Inputs []*Pipeline_Pair `json:"inputs,omitempty" cborgen:"inputs,omitempty"` ··· 61 Ref string `json:"ref" cborgen:"ref"` 62 } 63 64 - // Pipeline_Step is a "step" in the sh.tangled.pipeline schema. 65 - type Pipeline_Step struct { 66 - Command string `json:"command" cborgen:"command"` 67 - Environment []*Pipeline_Pair `json:"environment,omitempty" cborgen:"environment,omitempty"` 68 - Name string `json:"name" cborgen:"name"` 69 - } 70 - 71 // Pipeline_TriggerMetadata is a "triggerMetadata" in the sh.tangled.pipeline schema. 72 type Pipeline_TriggerMetadata struct { 73 Kind string `json:"kind" cborgen:"kind"` ··· 87 88 // Pipeline_Workflow is a "workflow" in the sh.tangled.pipeline schema. 89 type Pipeline_Workflow struct { 90 - Clone *Pipeline_CloneOpts `json:"clone" cborgen:"clone"` 91 - Dependencies []*Pipeline_Dependency `json:"dependencies" cborgen:"dependencies"` 92 - Environment []*Pipeline_Pair `json:"environment" cborgen:"environment"` 93 - Name string `json:"name" cborgen:"name"` 94 - Steps []*Pipeline_Step `json:"steps" cborgen:"steps"` 95 }

··· 29 Submodules bool `json:"submodules" cborgen:"submodules"` 30 } 31 0 0 0 0 0 0 32 // Pipeline_ManualTriggerData is a "manualTriggerData" in the sh.tangled.pipeline schema. 33 type Pipeline_ManualTriggerData struct { 34 Inputs []*Pipeline_Pair `json:"inputs,omitempty" cborgen:"inputs,omitempty"` ··· 55 Ref string `json:"ref" cborgen:"ref"` 56 } 57 0 0 0 0 0 0 0 58 // Pipeline_TriggerMetadata is a "triggerMetadata" in the sh.tangled.pipeline schema. 59 type Pipeline_TriggerMetadata struct { 60 Kind string `json:"kind" cborgen:"kind"` ··· 74 75 // Pipeline_Workflow is a "workflow" in the sh.tangled.pipeline schema. 76 type Pipeline_Workflow struct { 77 + Clone *Pipeline_CloneOpts `json:"clone" cborgen:"clone"` 78 + Engine string `json:"engine" cborgen:"engine"` 79 + Name string `json:"name" cborgen:"name"` 80 + Raw string `json:"raw" cborgen:"raw"` 0 81 }

-2

cmd/gen.go

··· 27 tangled.KnotMember{}, 28 tangled.Pipeline{}, 29 tangled.Pipeline_CloneOpts{}, 30 - tangled.Pipeline_Dependency{}, 31 tangled.Pipeline_ManualTriggerData{}, 32 tangled.Pipeline_Pair{}, 33 tangled.Pipeline_PullRequestTriggerData{}, 34 tangled.Pipeline_PushTriggerData{}, 35 tangled.PipelineStatus{}, 36 - tangled.Pipeline_Step{}, 37 tangled.Pipeline_TriggerMetadata{}, 38 tangled.Pipeline_TriggerRepo{}, 39 tangled.Pipeline_Workflow{},

··· 27 tangled.KnotMember{}, 28 tangled.Pipeline{}, 29 tangled.Pipeline_CloneOpts{}, 0 30 tangled.Pipeline_ManualTriggerData{}, 31 tangled.Pipeline_Pair{}, 32 tangled.Pipeline_PullRequestTriggerData{}, 33 tangled.Pipeline_PushTriggerData{}, 34 tangled.PipelineStatus{}, 0 35 tangled.Pipeline_TriggerMetadata{}, 36 tangled.Pipeline_TriggerRepo{}, 37 tangled.Pipeline_Workflow{},

+26 -3

docs/spindle/pipeline.md

··· 4 repo. Generally: 5 6 * Pipelines are defined in YAML. 7 - * Dependencies can be specified from 8 - [Nixpkgs](https://search.nixos.org) or custom registries. 9 - * Environment variables can be set globally or per-step. 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 10 11 Here's an example that uses all fields: 12

··· 4 repo. Generally: 5 6 * Pipelines are defined in YAML. 7 + * Workflows can run using different *engines*. 8 + 9 + The most barebones workflow looks like this: 10 + 11 + ```yaml 12 + when: 13 + - event: ["push"] 14 + branch: ["main"] 15 + 16 + engine: "nixery" 17 + 18 + # optional 19 + clone: 20 + skip: false 21 + depth: 50 22 + submodules: true 23 + ``` 24 + 25 + The `when` and `engine` fields are required, while every other aspect 26 + of how the definition is parsed is up to the engine. Currently, a spindle 27 + provides at least one of these built-in engines: 28 + 29 + ## `nixery` 30 + 31 + The Nixery engine uses an instance of [Nixery](https://nixery.dev) to run 32 + steps that use dependencies from [Nixpkgs](https://github.com/NixOS/nixpkgs). 33 34 Here's an example that uses all fields: 35

+7 -63

lexicons/pipeline/pipeline.json

··· 149 "type": "object", 150 "required": [ 151 "name", 152 - "dependencies", 153 - "steps", 154 - "environment", 155 - "clone" 156 ], 157 "properties": { 158 "name": { 159 "type": "string" 160 }, 161 - "dependencies": { 162 - "type": "array", 163 - "items": { 164 - "type": "ref", 165 - "ref": "#dependency" 166 - } 167 - }, 168 - "steps": { 169 - "type": "array", 170 - "items": { 171 - "type": "ref", 172 - "ref": "#step" 173 - } 174 - }, 175 - "environment": { 176 - "type": "array", 177 - "items": { 178 - "type": "ref", 179 - "ref": "#pair" 180 - } 181 }, 182 "clone": { 183 "type": "ref", 184 "ref": "#cloneOpts" 185 - } 186 - } 187 - }, 188 - "dependency": { 189 - "type": "object", 190 - "required": [ 191 - "registry", 192 - "packages" 193 - ], 194 - "properties": { 195 - "registry": { 196 "type": "string" 197 - }, 198 - "packages": { 199 - "type": "array", 200 - "items": { 201 - "type": "string" 202 - } 203 } 204 } 205 }, ··· 219 }, 220 "submodules": { 221 "type": "boolean" 222 - } 223 - } 224 - }, 225 - "step": { 226 - "type": "object", 227 - "required": [ 228 - "name", 229 - "command" 230 - ], 231 - "properties": { 232 - "name": { 233 - "type": "string" 234 - }, 235 - "command": { 236 - "type": "string" 237 - }, 238 - "environment": { 239 - "type": "array", 240 - "items": { 241 - "type": "ref", 242 - "ref": "#pair" 243 - } 244 } 245 } 246 },

··· 149 "type": "object", 150 "required": [ 151 "name", 152 + "engine", 153 + "clone", 154 + "raw" 0 155 ], 156 "properties": { 157 "name": { 158 "type": "string" 159 }, 160 + "engine": { 161 + "type": "string" 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 162 }, 163 "clone": { 164 "type": "ref", 165 "ref": "#cloneOpts" 166 + }, 167 + "raw": { 0 0 0 0 0 0 0 0 0 168 "type": "string" 0 0 0 0 0 0 169 } 170 } 171 }, ··· 185 }, 186 "submodules": { 187 "type": "boolean" 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 188 } 189 } 190 },

+2 -2

nix/modules/spindle.nix

··· 111 "SPINDLE_SERVER_SECRETS_PROVIDER=${cfg.server.secrets.provider}" 112 "SPINDLE_SERVER_SECRETS_OPENBAO_PROXY_ADDR=${cfg.server.secrets.openbao.proxyAddr}" 113 "SPINDLE_SERVER_SECRETS_OPENBAO_MOUNT=${cfg.server.secrets.openbao.mount}" 114 - "SPINDLE_PIPELINES_NIXERY=${cfg.pipelines.nixery}" 115 - "SPINDLE_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}" 116 ]; 117 ExecStart = "${cfg.package}/bin/spindle"; 118 Restart = "always";

··· 111 "SPINDLE_SERVER_SECRETS_PROVIDER=${cfg.server.secrets.provider}" 112 "SPINDLE_SERVER_SECRETS_OPENBAO_PROXY_ADDR=${cfg.server.secrets.openbao.proxyAddr}" 113 "SPINDLE_SERVER_SECRETS_OPENBAO_MOUNT=${cfg.server.secrets.openbao.mount}" 114 + "SPINDLE_NIXERY_PIPELINES_NIXERY=${cfg.pipelines.nixery}" 115 + "SPINDLE_NIXERY_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}" 116 ]; 117 ExecStart = "${cfg.package}/bin/spindle"; 118 Restart = "always";

+4 -4

spindle/config/config.go

··· 16 Dev bool `env:"DEV, default=false"` 17 Owner string `env:"OWNER, required"` 18 Secrets Secrets `env:",prefix=SECRETS_"` 0 19 } 20 21 func (s Server) Did() syntax.DID { ··· 32 Mount string `env:"MOUNT, default=spindle"` 33 } 34 35 - type Pipelines struct { 36 Nixery string `env:"NIXERY, default=nixery.tangled.sh"` 37 WorkflowTimeout string `env:"WORKFLOW_TIMEOUT, default=5m"` 38 - LogDir string `env:"LOG_DIR, default=/var/log/spindle"` 39 } 40 41 type Config struct { 42 - Server Server `env:",prefix=SPINDLE_SERVER_"` 43 - Pipelines Pipelines `env:",prefix=SPINDLE_PIPELINES_"` 44 } 45 46 func Load(ctx context.Context) (*Config, error) {

··· 16 Dev bool `env:"DEV, default=false"` 17 Owner string `env:"OWNER, required"` 18 Secrets Secrets `env:",prefix=SECRETS_"` 19 + LogDir string `env:"LOG_DIR, default=/var/log/spindle"` 20 } 21 22 func (s Server) Did() syntax.DID { ··· 33 Mount string `env:"MOUNT, default=spindle"` 34 } 35 36 + type NixeryPipelines struct { 37 Nixery string `env:"NIXERY, default=nixery.tangled.sh"` 38 WorkflowTimeout string `env:"WORKFLOW_TIMEOUT, default=5m"` 0 39 } 40 41 type Config struct { 42 + Server Server `env:",prefix=SPINDLE_SERVER_"` 43 + NixeryPipelines NixeryPipelines `env:",prefix=SPINDLE_NIXERY_PIPELINES_"` 44 } 45 46 func Load(ctx context.Context) (*Config, error) {

+1 -1

spindle/engine/ansi_stripper.go spindle/engines/nixery/ansi_stripper.go

··· 1 - package engine 2 3 import ( 4 "io"

··· 1 + package nixery 2 3 import ( 4 "io"

+64 -416

spindle/engine/engine.go

··· 4 "context" 5 "errors" 6 "fmt" 7 - "io" 8 "log/slog" 9 - "os" 10 - "strings" 11 - "sync" 12 - "time" 13 14 securejoin "github.com/cyphar/filepath-securejoin" 15 - "github.com/docker/docker/api/types/container" 16 - "github.com/docker/docker/api/types/image" 17 - "github.com/docker/docker/api/types/mount" 18 - "github.com/docker/docker/api/types/network" 19 - "github.com/docker/docker/api/types/volume" 20 - "github.com/docker/docker/client" 21 - "github.com/docker/docker/pkg/stdcopy" 22 "golang.org/x/sync/errgroup" 23 - "tangled.sh/tangled.sh/core/log" 24 "tangled.sh/tangled.sh/core/notifier" 25 "tangled.sh/tangled.sh/core/spindle/config" 26 "tangled.sh/tangled.sh/core/spindle/db" ··· 28 "tangled.sh/tangled.sh/core/spindle/secrets" 29 ) 30 31 - const ( 32 - workspaceDir = "/tangled/workspace" 0 33 ) 34 35 - type cleanupFunc func(context.Context) error 36 - 37 - type Engine struct { 38 - docker client.APIClient 39 - l *slog.Logger 40 - db *db.DB 41 - n *notifier.Notifier 42 - cfg *config.Config 43 - vault secrets.Manager 44 - 45 - cleanupMu sync.Mutex 46 - cleanup map[string][]cleanupFunc 47 - } 48 - 49 - func New(ctx context.Context, cfg *config.Config, db *db.DB, n *notifier.Notifier, vault secrets.Manager) (*Engine, error) { 50 - dcli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) 51 - if err != nil { 52 - return nil, err 53 - } 54 - 55 - l := log.FromContext(ctx).With("component", "spindle") 56 - 57 - e := &Engine{ 58 - docker: dcli, 59 - l: l, 60 - db: db, 61 - n: n, 62 - cfg: cfg, 63 - vault: vault, 64 - } 65 - 66 - e.cleanup = make(map[string][]cleanupFunc) 67 - 68 - return e, nil 69 - } 70 - 71 - func (e *Engine) StartWorkflows(ctx context.Context, pipeline *models.Pipeline, pipelineId models.PipelineId) { 72 - e.l.Info("starting all workflows in parallel", "pipeline", pipelineId) 73 74 // extract secrets 75 var allSecrets []secrets.UnlockedSecret 76 if didSlashRepo, err := securejoin.SecureJoin(pipeline.RepoOwner, pipeline.RepoName); err == nil { 77 - if res, err := e.vault.GetSecretsUnlocked(ctx, secrets.DidSlashRepo(didSlashRepo)); err == nil { 78 allSecrets = res 79 } 80 } 81 82 - workflowTimeoutStr := e.cfg.Pipelines.WorkflowTimeout 83 - workflowTimeout, err := time.ParseDuration(workflowTimeoutStr) 84 - if err != nil { 85 - e.l.Error("failed to parse workflow timeout", "error", err, "timeout", workflowTimeoutStr) 86 - workflowTimeout = 5 * time.Minute 87 - } 88 - e.l.Info("using workflow timeout", "timeout", workflowTimeout) 89 - 90 eg, ctx := errgroup.WithContext(ctx) 91 - for _, w := range pipeline.Workflows { 92 - eg.Go(func() error { 93 - wid := models.WorkflowId{ 94 - PipelineId: pipelineId, 95 - Name: w.Name, 96 - } 97 - 98 - err := e.db.StatusRunning(wid, e.n) 99 - if err != nil { 100 - return err 101 - } 102 103 - err = e.SetupWorkflow(ctx, wid) 104 - if err != nil { 105 - e.l.Error("setting up worklow", "wid", wid, "err", err) 106 - return err 107 - } 108 - defer e.DestroyWorkflow(ctx, wid) 109 - 110 - reader, err := e.docker.ImagePull(ctx, w.Image, image.PullOptions{}) 111 - if err != nil { 112 - e.l.Error("pipeline image pull failed!", "image", w.Image, "workflowId", wid, "error", err.Error()) 113 114 - err := e.db.StatusFailed(wid, err.Error(), -1, e.n) 115 if err != nil { 116 return err 117 } 118 119 - return fmt.Errorf("pulling image: %w", err) 120 - } 121 - defer reader.Close() 122 - io.Copy(os.Stdout, reader) 123 - 124 - ctx, cancel := context.WithTimeout(ctx, workflowTimeout) 125 - defer cancel() 126 127 - err = e.StartSteps(ctx, wid, w, allSecrets) 128 - if err != nil { 129 - if errors.Is(err, ErrTimedOut) { 130 - dbErr := e.db.StatusTimeout(wid, e.n) 131 - if dbErr != nil { 132 - return dbErr 133 - } 134 - } else { 135 - dbErr := e.db.StatusFailed(wid, err.Error(), -1, e.n) 136 if dbErr != nil { 137 return dbErr 138 } 0 139 } 140 - 141 - return fmt.Errorf("starting steps image: %w", err) 142 - } 143 - 144 - err = e.db.StatusSuccess(wid, e.n) 145 - if err != nil { 146 - return err 147 - } 148 - 149 - return nil 150 - }) 151 - } 152 - 153 - if err = eg.Wait(); err != nil { 154 - e.l.Error("failed to run one or more workflows", "err", err) 155 - } else { 156 - e.l.Error("successfully ran full pipeline") 157 - } 158 - } 159 - 160 - // SetupWorkflow sets up a new network for the workflow and volumes for 161 - // the workspace and Nix store. These are persisted across steps and are 162 - // destroyed at the end of the workflow. 163 - func (e *Engine) SetupWorkflow(ctx context.Context, wid models.WorkflowId) error { 164 - e.l.Info("setting up workflow", "workflow", wid) 165 - 166 - _, err := e.docker.VolumeCreate(ctx, volume.CreateOptions{ 167 - Name: workspaceVolume(wid), 168 - Driver: "local", 169 - }) 170 - if err != nil { 171 - return err 172 - } 173 - e.registerCleanup(wid, func(ctx context.Context) error { 174 - return e.docker.VolumeRemove(ctx, workspaceVolume(wid), true) 175 - }) 176 - 177 - _, err = e.docker.VolumeCreate(ctx, volume.CreateOptions{ 178 - Name: nixVolume(wid), 179 - Driver: "local", 180 - }) 181 - if err != nil { 182 - return err 183 - } 184 - e.registerCleanup(wid, func(ctx context.Context) error { 185 - return e.docker.VolumeRemove(ctx, nixVolume(wid), true) 186 - }) 187 - 188 - _, err = e.docker.NetworkCreate(ctx, networkName(wid), network.CreateOptions{ 189 - Driver: "bridge", 190 - }) 191 - if err != nil { 192 - return err 193 - } 194 - e.registerCleanup(wid, func(ctx context.Context) error { 195 - return e.docker.NetworkRemove(ctx, networkName(wid)) 196 - }) 197 - 198 - return nil 199 - } 200 - 201 - // StartSteps starts all steps sequentially with the same base image. 202 - // ONLY marks pipeline as failed if container's exit code is non-zero. 203 - // All other errors are bubbled up. 204 - // Fixed version of the step execution logic 205 - func (e *Engine) StartSteps(ctx context.Context, wid models.WorkflowId, w models.Workflow, secrets []secrets.UnlockedSecret) error { 206 - workflowEnvs := ConstructEnvs(w.Environment) 207 - for _, s := range secrets { 208 - workflowEnvs.AddEnv(s.Key, s.Value) 209 - } 210 - 211 - for stepIdx, step := range w.Steps { 212 - select { 213 - case <-ctx.Done(): 214 - return ctx.Err() 215 - default: 216 - } 217 - 218 - envs := append(EnvVars(nil), workflowEnvs...) 219 - for k, v := range step.Environment { 220 - envs.AddEnv(k, v) 221 - } 222 - envs.AddEnv("HOME", workspaceDir) 223 - e.l.Debug("envs for step", "step", step.Name, "envs", envs.Slice()) 224 - 225 - hostConfig := hostConfig(wid) 226 - resp, err := e.docker.ContainerCreate(ctx, &container.Config{ 227 - Image: w.Image, 228 - Cmd: []string{"bash", "-c", step.Command}, 229 - WorkingDir: workspaceDir, 230 - Tty: false, 231 - Hostname: "spindle", 232 - Env: envs.Slice(), 233 - }, hostConfig, nil, nil, "") 234 - defer e.DestroyStep(ctx, resp.ID) 235 - if err != nil { 236 - return fmt.Errorf("creating container: %w", err) 237 - } 238 - 239 - err = e.docker.NetworkConnect(ctx, networkName(wid), resp.ID, nil) 240 - if err != nil { 241 - return fmt.Errorf("connecting network: %w", err) 242 - } 243 - 244 - err = e.docker.ContainerStart(ctx, resp.ID, container.StartOptions{}) 245 - if err != nil { 246 - return err 247 - } 248 - e.l.Info("started container", "name", resp.ID, "step", step.Name) 249 - 250 - // start tailing logs in background 251 - tailDone := make(chan error, 1) 252 - go func() { 253 - tailDone <- e.TailStep(ctx, resp.ID, wid, stepIdx, step) 254 - }() 255 - 256 - // wait for container completion or timeout 257 - waitDone := make(chan struct{}) 258 - var state *container.State 259 - var waitErr error 260 - 261 - go func() { 262 - defer close(waitDone) 263 - state, waitErr = e.WaitStep(ctx, resp.ID) 264 - }() 265 - 266 - select { 267 - case <-waitDone: 268 - 269 - // wait for tailing to complete 270 - <-tailDone 271 - 272 - case <-ctx.Done(): 273 - e.l.Warn("step timed out; killing container", "container", resp.ID, "step", step.Name) 274 - err = e.DestroyStep(context.Background(), resp.ID) 275 - if err != nil { 276 - e.l.Error("failed to destroy step", "container", resp.ID, "error", err) 277 - } 278 - 279 - // wait for both goroutines to finish 280 - <-waitDone 281 - <-tailDone 282 - 283 - return ErrTimedOut 284 - } 285 - 286 - select { 287 - case <-ctx.Done(): 288 - return ctx.Err() 289 - default: 290 - } 291 - 292 - if waitErr != nil { 293 - return waitErr 294 - } 295 - 296 - err = e.DestroyStep(ctx, resp.ID) 297 - if err != nil { 298 - return err 299 - } 300 - 301 - if state.ExitCode != 0 { 302 - e.l.Error("workflow failed!", "workflow_id", wid.String(), "error", state.Error, "exit_code", state.ExitCode, "oom_killed", state.OOMKilled) 303 - if state.OOMKilled { 304 - return ErrOOMKilled 305 - } 306 - return ErrWorkflowFailed 307 - } 308 - } 309 - 310 - return nil 311 - } 312 - 313 - func (e *Engine) WaitStep(ctx context.Context, containerID string) (*container.State, error) { 314 - wait, errCh := e.docker.ContainerWait(ctx, containerID, container.WaitConditionNotRunning) 315 - select { 316 - case err := <-errCh: 317 - if err != nil { 318 - return nil, err 319 - } 320 - case <-wait: 321 - } 322 - 323 - e.l.Info("waited for container", "name", containerID) 324 - 325 - info, err := e.docker.ContainerInspect(ctx, containerID) 326 - if err != nil { 327 - return nil, err 328 - } 329 - 330 - return info.State, nil 331 - } 332 - 333 - func (e *Engine) TailStep(ctx context.Context, containerID string, wid models.WorkflowId, stepIdx int, step models.Step) error { 334 - wfLogger, err := NewWorkflowLogger(e.cfg.Pipelines.LogDir, wid) 335 - if err != nil { 336 - e.l.Warn("failed to setup step logger; logs will not be persisted", "error", err) 337 - return err 338 - } 339 - defer wfLogger.Close() 340 - 341 - ctl := wfLogger.ControlWriter(stepIdx, step) 342 - ctl.Write([]byte(step.Name)) 343 - 344 - logs, err := e.docker.ContainerLogs(ctx, containerID, container.LogsOptions{ 345 - Follow: true, 346 - ShowStdout: true, 347 - ShowStderr: true, 348 - Details: false, 349 - Timestamps: false, 350 - }) 351 - if err != nil { 352 - return err 353 - } 354 - 355 - _, err = stdcopy.StdCopy( 356 - wfLogger.DataWriter("stdout"), 357 - wfLogger.DataWriter("stderr"), 358 - logs, 359 - ) 360 - if err != nil && err != io.EOF && !errors.Is(err, context.DeadlineExceeded) { 361 - return fmt.Errorf("failed to copy logs: %w", err) 362 - } 363 364 - return nil 365 - } 0 0 0 0 0 366 367 - func (e *Engine) DestroyStep(ctx context.Context, containerID string) error { 368 - err := e.docker.ContainerKill(ctx, containerID, "9") // SIGKILL 369 - if err != nil && !isErrContainerNotFoundOrNotRunning(err) { 370 - return err 371 - } 372 373 - if err := e.docker.ContainerRemove(ctx, containerID, container.RemoveOptions{ 374 - RemoveVolumes: true, 375 - RemoveLinks: false, 376 - Force: false, 377 - }); err != nil && !isErrContainerNotFoundOrNotRunning(err) { 378 - return err 379 - } 380 381 - return nil 382 - } 0 0 0 0 0 0 0 0 0 0 0 383 384 - func (e *Engine) DestroyWorkflow(ctx context.Context, wid models.WorkflowId) error { 385 - e.cleanupMu.Lock() 386 - key := wid.String() 387 388 - fns := e.cleanup[key] 389 - delete(e.cleanup, key) 390 - e.cleanupMu.Unlock() 0 391 392 - for _, fn := range fns { 393 - if err := fn(ctx); err != nil { 394 - e.l.Error("failed to cleanup workflow resource", "workflowId", wid, "error", err) 395 } 396 } 397 - return nil 398 - } 399 400 - func (e *Engine) registerCleanup(wid models.WorkflowId, fn cleanupFunc) { 401 - e.cleanupMu.Lock() 402 - defer e.cleanupMu.Unlock() 403 - 404 - key := wid.String() 405 - e.cleanup[key] = append(e.cleanup[key], fn) 406 - } 407 - 408 - func workspaceVolume(wid models.WorkflowId) string { 409 - return fmt.Sprintf("workspace-%s", wid) 410 - } 411 - 412 - func nixVolume(wid models.WorkflowId) string { 413 - return fmt.Sprintf("nix-%s", wid) 414 - } 415 - 416 - func networkName(wid models.WorkflowId) string { 417 - return fmt.Sprintf("workflow-network-%s", wid) 418 - } 419 - 420 - func hostConfig(wid models.WorkflowId) *container.HostConfig { 421 - hostConfig := &container.HostConfig{ 422 - Mounts: []mount.Mount{ 423 - { 424 - Type: mount.TypeVolume, 425 - Source: workspaceVolume(wid), 426 - Target: workspaceDir, 427 - }, 428 - { 429 - Type: mount.TypeVolume, 430 - Source: nixVolume(wid), 431 - Target: "/nix", 432 - }, 433 - { 434 - Type: mount.TypeTmpfs, 435 - Target: "/tmp", 436 - ReadOnly: false, 437 - TmpfsOptions: &mount.TmpfsOptions{ 438 - Mode: 0o1777, // world-writeable sticky bit 439 - Options: [][]string{ 440 - {"exec"}, 441 - }, 442 - }, 443 - }, 444 - { 445 - Type: mount.TypeVolume, 446 - Source: "etc-nix-" + wid.String(), 447 - Target: "/etc/nix", 448 - }, 449 - }, 450 - ReadonlyRootfs: false, 451 - CapDrop: []string{"ALL"}, 452 - CapAdd: []string{"CAP_DAC_OVERRIDE"}, 453 - SecurityOpt: []string{"no-new-privileges"}, 454 - ExtraHosts: []string{"host.docker.internal:host-gateway"}, 455 } 456 - 457 - return hostConfig 458 - } 459 - 460 - // thanks woodpecker 461 - func isErrContainerNotFoundOrNotRunning(err error) bool { 462 - // Error response from daemon: Cannot kill container: ...: No such container: ... 463 - // Error response from daemon: Cannot kill container: ...: Container ... is not running" 464 - // Error response from podman daemon: can only kill running containers. ... is in state exited 465 - // Error: No such container: ... 466 - return err != nil && (strings.Contains(err.Error(), "No such container") || strings.Contains(err.Error(), "is not running") || strings.Contains(err.Error(), "can only kill running containers")) 467 }

··· 4 "context" 5 "errors" 6 "fmt" 0 7 "log/slog" 0 0 0 0 8 9 securejoin "github.com/cyphar/filepath-securejoin" 0 0 0 0 0 0 0 10 "golang.org/x/sync/errgroup" 0 11 "tangled.sh/tangled.sh/core/notifier" 12 "tangled.sh/tangled.sh/core/spindle/config" 13 "tangled.sh/tangled.sh/core/spindle/db" ··· 15 "tangled.sh/tangled.sh/core/spindle/secrets" 16 ) 17 18 + var ( 19 + ErrTimedOut = errors.New("timed out") 20 + ErrWorkflowFailed = errors.New("workflow failed") 21 ) 22 23 + func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, db *db.DB, n *notifier.Notifier, ctx context.Context, pipeline *models.Pipeline, pipelineId models.PipelineId) { 24 + l.Info("starting all workflows in parallel", "pipeline", pipelineId) 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 25 26 // extract secrets 27 var allSecrets []secrets.UnlockedSecret 28 if didSlashRepo, err := securejoin.SecureJoin(pipeline.RepoOwner, pipeline.RepoName); err == nil { 29 + if res, err := vault.GetSecretsUnlocked(ctx, secrets.DidSlashRepo(didSlashRepo)); err == nil { 30 allSecrets = res 31 } 32 } 33 0 0 0 0 0 0 0 0 34 eg, ctx := errgroup.WithContext(ctx) 35 + for eng, wfs := range pipeline.Workflows { 36 + workflowTimeout := eng.WorkflowTimeout() 37 + l.Info("using workflow timeout", "timeout", workflowTimeout) 0 0 0 0 0 0 0 0 38 39 + for _, w := range wfs { 40 + eg.Go(func() error { 41 + wid := models.WorkflowId{ 42 + PipelineId: pipelineId, 43 + Name: w.Name, 44 + } 0 0 0 0 45 46 + err := db.StatusRunning(wid, n) 47 if err != nil { 48 return err 49 } 50 51 + err = eng.SetupWorkflow(ctx, wid, &w) 52 + if err != nil { 53 + // TODO(winter): Should this always set StatusFailed? 54 + // In the original, we only do in a subset of cases. 55 + l.Error("setting up worklow", "wid", wid, "err", err) 0 0 56 57 + dbErr := db.StatusFailed(wid, err.Error(), -1, n) 0 0 0 0 0 0 0 0 58 if dbErr != nil { 59 return dbErr 60 } 61 + return err 62 } 63 + defer eng.DestroyWorkflow(ctx, wid) 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 64 65 + wfLogger, err := models.NewWorkflowLogger(cfg.Server.LogDir, wid) 66 + if err != nil { 67 + l.Warn("failed to setup step logger; logs will not be persisted", "error", err) 68 + wfLogger = nil 69 + } else { 70 + defer wfLogger.Close() 71 + } 72 73 + ctx, cancel := context.WithTimeout(ctx, workflowTimeout) 74 + defer cancel() 0 0 0 75 76 + for stepIdx, step := range w.Steps { 77 + if wfLogger != nil { 78 + ctl := wfLogger.ControlWriter(stepIdx, step) 79 + ctl.Write([]byte(step.Name())) 80 + } 0 0 81 82 + err = eng.RunStep(ctx, wid, &w, stepIdx, allSecrets, wfLogger) 83 + if err != nil { 84 + if errors.Is(err, ErrTimedOut) { 85 + dbErr := db.StatusTimeout(wid, n) 86 + if dbErr != nil { 87 + return dbErr 88 + } 89 + } else { 90 + dbErr := db.StatusFailed(wid, err.Error(), -1, n) 91 + if dbErr != nil { 92 + return dbErr 93 + } 94 + } 95 96 + return fmt.Errorf("starting steps image: %w", err) 97 + } 98 + } 99 100 + err = db.StatusSuccess(wid, n) 101 + if err != nil { 102 + return err 103 + } 104 105 + return nil 106 + }) 0 107 } 108 } 0 0 109 110 + if err := eg.Wait(); err != nil { 111 + l.Error("failed to run one or more workflows", "err", err) 112 + } else { 113 + l.Error("successfully ran full pipeline") 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 114 } 0 0 0 0 0 0 0 0 0 0 0 115 }

+1 -1

spindle/engine/envs.go spindle/engines/nixery/envs.go

··· 1 - package engine 2 3 import ( 4 "fmt"

··· 1 + package nixery 2 3 import ( 4 "fmt"

+1 -1

spindle/engine/envs_test.go spindle/engines/nixery/envs_test.go

··· 1 - package engine 2 3 import ( 4 "testing"

··· 1 + package nixery 2 3 import ( 4 "testing"

-9

spindle/engine/errors.go

··· 1 - package engine 2 - 3 - import "errors" 4 - 5 - var ( 6 - ErrOOMKilled = errors.New("oom killed") 7 - ErrTimedOut = errors.New("timed out") 8 - ErrWorkflowFailed = errors.New("workflow failed") 9 - )

··· 0 0 0 0 0 0 0 0 0

+8 -10

spindle/engine/logger.go spindle/models/logger.go

··· 1 - package engine 2 3 import ( 4 "encoding/json" ··· 7 "os" 8 "path/filepath" 9 "strings" 10 - 11 - "tangled.sh/tangled.sh/core/spindle/models" 12 ) 13 14 type WorkflowLogger struct { ··· 16 encoder *json.Encoder 17 } 18 19 - func NewWorkflowLogger(baseDir string, wid models.WorkflowId) (*WorkflowLogger, error) { 20 path := LogFilePath(baseDir, wid) 21 22 file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) ··· 30 }, nil 31 } 32 33 - func LogFilePath(baseDir string, workflowID models.WorkflowId) string { 34 logFilePath := filepath.Join(baseDir, fmt.Sprintf("%s.log", workflowID.String())) 35 return logFilePath 36 } ··· 47 } 48 } 49 50 - func (l *WorkflowLogger) ControlWriter(idx int, step models.Step) io.Writer { 51 return &controlWriter{ 52 logger: l, 53 idx: idx, ··· 62 63 func (w *dataWriter) Write(p []byte) (int, error) { 64 line := strings.TrimRight(string(p), "\r\n") 65 - entry := models.NewDataLogLine(line, w.stream) 66 if err := w.logger.encoder.Encode(entry); err != nil { 67 return 0, err 68 } ··· 72 type controlWriter struct { 73 logger *WorkflowLogger 74 idx int 75 - step models.Step 76 } 77 78 func (w *controlWriter) Write(_ []byte) (int, error) { 79 - entry := models.NewControlLogLine(w.idx, w.step) 80 if err := w.logger.encoder.Encode(entry); err != nil { 81 return 0, err 82 } 83 - return len(w.step.Name), nil 84 }

··· 1 + package models 2 3 import ( 4 "encoding/json" ··· 7 "os" 8 "path/filepath" 9 "strings" 0 0 10 ) 11 12 type WorkflowLogger struct { ··· 14 encoder *json.Encoder 15 } 16 17 + func NewWorkflowLogger(baseDir string, wid WorkflowId) (*WorkflowLogger, error) { 18 path := LogFilePath(baseDir, wid) 19 20 file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) ··· 28 }, nil 29 } 30 31 + func LogFilePath(baseDir string, workflowID WorkflowId) string { 32 logFilePath := filepath.Join(baseDir, fmt.Sprintf("%s.log", workflowID.String())) 33 return logFilePath 34 } ··· 45 } 46 } 47 48 + func (l *WorkflowLogger) ControlWriter(idx int, step Step) io.Writer { 49 return &controlWriter{ 50 logger: l, 51 idx: idx, ··· 60 61 func (w *dataWriter) Write(p []byte) (int, error) { 62 line := strings.TrimRight(string(p), "\r\n") 63 + entry := NewDataLogLine(line, w.stream) 64 if err := w.logger.encoder.Encode(entry); err != nil { 65 return 0, err 66 } ··· 70 type controlWriter struct { 71 logger *WorkflowLogger 72 idx int 73 + step Step 74 } 75 76 func (w *controlWriter) Write(_ []byte) (int, error) { 77 + entry := NewControlLogLine(w.idx, w.step) 78 if err := w.logger.encoder.Encode(entry); err != nil { 79 return 0, err 80 } 81 + return len(w.step.Name()), nil 82 }

+476

spindle/engines/nixery/engine.go

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

··· 1 + package nixery 2 + 3 + import ( 4 + "context" 5 + "errors" 6 + "fmt" 7 + "io" 8 + "log/slog" 9 + "os" 10 + "path" 11 + "strings" 12 + "sync" 13 + "time" 14 + 15 + "github.com/docker/docker/api/types/container" 16 + "github.com/docker/docker/api/types/image" 17 + "github.com/docker/docker/api/types/mount" 18 + "github.com/docker/docker/api/types/network" 19 + "github.com/docker/docker/api/types/volume" 20 + "github.com/docker/docker/client" 21 + "github.com/docker/docker/pkg/stdcopy" 22 + "gopkg.in/yaml.v3" 23 + "tangled.sh/tangled.sh/core/api/tangled" 24 + "tangled.sh/tangled.sh/core/log" 25 + "tangled.sh/tangled.sh/core/spindle/config" 26 + "tangled.sh/tangled.sh/core/spindle/engine" 27 + "tangled.sh/tangled.sh/core/spindle/models" 28 + "tangled.sh/tangled.sh/core/spindle/secrets" 29 + ) 30 + 31 + const ( 32 + workspaceDir = "/tangled/workspace" 33 + ) 34 + 35 + type cleanupFunc func(context.Context) error 36 + 37 + type Engine struct { 38 + docker client.APIClient 39 + l *slog.Logger 40 + cfg *config.Config 41 + 42 + cleanupMu sync.Mutex 43 + cleanup map[string][]cleanupFunc 44 + } 45 + 46 + type Step struct { 47 + name string 48 + kind models.StepKind 49 + command string 50 + environment map[string]string 51 + } 52 + 53 + func (s Step) Name() string { 54 + return s.name 55 + } 56 + 57 + func (s Step) Command() string { 58 + return s.command 59 + } 60 + 61 + func (s Step) Kind() models.StepKind { 62 + return s.kind 63 + } 64 + 65 + // setupSteps get added to start of Steps 66 + type setupSteps []models.Step 67 + 68 + // addStep adds a step to the beginning of the workflow's steps. 69 + func (ss *setupSteps) addStep(step models.Step) { 70 + *ss = append(*ss, step) 71 + } 72 + 73 + type addlFields struct { 74 + image string 75 + env map[string]string 76 + } 77 + 78 + func (e *Engine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipeline) (*models.Workflow, error) { 79 + swf := &models.Workflow{} 80 + addl := addlFields{} 81 + 82 + dwf := &struct { 83 + Steps []struct { 84 + Command string `yaml:"command"` 85 + Name string `yaml:"name"` 86 + Environment map[string]string `yaml:"environment"` 87 + } `yaml:"steps"` 88 + Dependencies map[string][]string `yaml:"dependencies"` 89 + Environment map[string]string `yaml:"environment"` 90 + }{} 91 + err := yaml.Unmarshal([]byte(twf.Raw), &dwf) 92 + if err != nil { 93 + return nil, err 94 + } 95 + 96 + for _, dstep := range dwf.Steps { 97 + sstep := Step{} 98 + sstep.environment = dstep.Environment 99 + sstep.command = dstep.Command 100 + sstep.name = dstep.Name 101 + sstep.kind = models.StepKindUser 102 + swf.Steps = append(swf.Steps, sstep) 103 + } 104 + swf.Name = twf.Name 105 + addl.env = dwf.Environment 106 + addl.image = workflowImage(dwf.Dependencies, e.cfg.NixeryPipelines.Nixery) 107 + 108 + setup := &setupSteps{} 109 + 110 + setup.addStep(nixConfStep()) 111 + setup.addStep(cloneStep(twf, *tpl.TriggerMetadata, e.cfg.Server.Dev)) 112 + // this step could be empty 113 + if s := dependencyStep(dwf.Dependencies); s != nil { 114 + setup.addStep(*s) 115 + } 116 + 117 + // append setup steps in order to the start of workflow steps 118 + swf.Steps = append(*setup, swf.Steps...) 119 + swf.Data = addl 120 + 121 + return swf, nil 122 + } 123 + 124 + func (e *Engine) WorkflowTimeout() time.Duration { 125 + workflowTimeoutStr := e.cfg.NixeryPipelines.WorkflowTimeout 126 + workflowTimeout, err := time.ParseDuration(workflowTimeoutStr) 127 + if err != nil { 128 + e.l.Error("failed to parse workflow timeout", "error", err, "timeout", workflowTimeoutStr) 129 + workflowTimeout = 5 * time.Minute 130 + } 131 + 132 + return workflowTimeout 133 + } 134 + 135 + func workflowImage(deps map[string][]string, nixery string) string { 136 + var dependencies string 137 + for reg, ds := range deps { 138 + if reg == "nixpkgs" { 139 + dependencies = path.Join(ds...) 140 + } 141 + } 142 + 143 + // load defaults from somewhere else 144 + dependencies = path.Join(dependencies, "bash", "git", "coreutils", "nix") 145 + 146 + return path.Join(nixery, dependencies) 147 + } 148 + 149 + func New(ctx context.Context, cfg *config.Config) (*Engine, error) { 150 + dcli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) 151 + if err != nil { 152 + return nil, err 153 + } 154 + 155 + l := log.FromContext(ctx).With("component", "spindle") 156 + 157 + e := &Engine{ 158 + docker: dcli, 159 + l: l, 160 + cfg: cfg, 161 + } 162 + 163 + e.cleanup = make(map[string][]cleanupFunc) 164 + 165 + return e, nil 166 + } 167 + 168 + // SetupWorkflow sets up a new network for the workflow and volumes for 169 + // the workspace and Nix store. These are persisted across steps and are 170 + // destroyed at the end of the workflow. 171 + func (e *Engine) SetupWorkflow(ctx context.Context, wid models.WorkflowId, wf *models.Workflow) error { 172 + e.l.Info("setting up workflow", "workflow", wid) 173 + 174 + _, err := e.docker.VolumeCreate(ctx, volume.CreateOptions{ 175 + Name: workspaceVolume(wid), 176 + Driver: "local", 177 + }) 178 + if err != nil { 179 + return err 180 + } 181 + e.registerCleanup(wid, func(ctx context.Context) error { 182 + return e.docker.VolumeRemove(ctx, workspaceVolume(wid), true) 183 + }) 184 + 185 + _, err = e.docker.VolumeCreate(ctx, volume.CreateOptions{ 186 + Name: nixVolume(wid), 187 + Driver: "local", 188 + }) 189 + if err != nil { 190 + return err 191 + } 192 + e.registerCleanup(wid, func(ctx context.Context) error { 193 + return e.docker.VolumeRemove(ctx, nixVolume(wid), true) 194 + }) 195 + 196 + _, err = e.docker.NetworkCreate(ctx, networkName(wid), network.CreateOptions{ 197 + Driver: "bridge", 198 + }) 199 + if err != nil { 200 + return err 201 + } 202 + e.registerCleanup(wid, func(ctx context.Context) error { 203 + return e.docker.NetworkRemove(ctx, networkName(wid)) 204 + }) 205 + 206 + addl := wf.Data.(addlFields) 207 + 208 + reader, err := e.docker.ImagePull(ctx, addl.image, image.PullOptions{}) 209 + if err != nil { 210 + e.l.Error("pipeline image pull failed!", "image", addl.image, "workflowId", wid, "error", err.Error()) 211 + 212 + return fmt.Errorf("pulling image: %w", err) 213 + } 214 + defer reader.Close() 215 + io.Copy(os.Stdout, reader) 216 + 217 + return nil 218 + } 219 + 220 + func (e *Engine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.Workflow, idx int, secrets []secrets.UnlockedSecret, wfLogger *models.WorkflowLogger) error { 221 + workflowEnvs := ConstructEnvs(w.Data.(addlFields).env) 222 + for _, s := range secrets { 223 + workflowEnvs.AddEnv(s.Key, s.Value) 224 + } 225 + 226 + step := w.Steps[idx].(Step) 227 + 228 + select { 229 + case <-ctx.Done(): 230 + return ctx.Err() 231 + default: 232 + } 233 + 234 + envs := append(EnvVars(nil), workflowEnvs...) 235 + for k, v := range step.environment { 236 + envs.AddEnv(k, v) 237 + } 238 + envs.AddEnv("HOME", workspaceDir) 239 + e.l.Debug("envs for step", "step", step.Name, "envs", envs.Slice()) 240 + 241 + hostConfig := hostConfig(wid) 242 + resp, err := e.docker.ContainerCreate(ctx, &container.Config{ 243 + Image: w.Data.(addlFields).image, 244 + Cmd: []string{"bash", "-c", step.command}, 245 + WorkingDir: workspaceDir, 246 + Tty: false, 247 + Hostname: "spindle", 248 + Env: envs.Slice(), 249 + }, hostConfig, nil, nil, "") 250 + defer e.DestroyStep(ctx, resp.ID) 251 + if err != nil { 252 + return fmt.Errorf("creating container: %w", err) 253 + } 254 + 255 + err = e.docker.NetworkConnect(ctx, networkName(wid), resp.ID, nil) 256 + if err != nil { 257 + return fmt.Errorf("connecting network: %w", err) 258 + } 259 + 260 + err = e.docker.ContainerStart(ctx, resp.ID, container.StartOptions{}) 261 + if err != nil { 262 + return err 263 + } 264 + e.l.Info("started container", "name", resp.ID, "step", step.Name) 265 + 266 + // start tailing logs in background 267 + tailDone := make(chan error, 1) 268 + go func() { 269 + tailDone <- e.tailStep(ctx, wfLogger, resp.ID, wid, idx, step) 270 + }() 271 + 272 + // wait for container completion or timeout 273 + waitDone := make(chan struct{}) 274 + var state *container.State 275 + var waitErr error 276 + 277 + go func() { 278 + defer close(waitDone) 279 + state, waitErr = e.WaitStep(ctx, resp.ID) 280 + }() 281 + 282 + select { 283 + case <-waitDone: 284 + 285 + // wait for tailing to complete 286 + <-tailDone 287 + 288 + case <-ctx.Done(): 289 + e.l.Warn("step timed out; killing container", "container", resp.ID, "step", step.Name) 290 + err = e.DestroyStep(context.Background(), resp.ID) 291 + if err != nil { 292 + e.l.Error("failed to destroy step", "container", resp.ID, "error", err) 293 + } 294 + 295 + // wait for both goroutines to finish 296 + <-waitDone 297 + <-tailDone 298 + 299 + return engine.ErrTimedOut 300 + } 301 + 302 + select { 303 + case <-ctx.Done(): 304 + return ctx.Err() 305 + default: 306 + } 307 + 308 + if waitErr != nil { 309 + return waitErr 310 + } 311 + 312 + err = e.DestroyStep(ctx, resp.ID) 313 + if err != nil { 314 + return err 315 + } 316 + 317 + if state.ExitCode != 0 { 318 + e.l.Error("workflow failed!", "workflow_id", wid.String(), "error", state.Error, "exit_code", state.ExitCode, "oom_killed", state.OOMKilled) 319 + if state.OOMKilled { 320 + return ErrOOMKilled 321 + } 322 + return engine.ErrWorkflowFailed 323 + } 324 + 325 + return nil 326 + } 327 + 328 + func (e *Engine) WaitStep(ctx context.Context, containerID string) (*container.State, error) { 329 + wait, errCh := e.docker.ContainerWait(ctx, containerID, container.WaitConditionNotRunning) 330 + select { 331 + case err := <-errCh: 332 + if err != nil { 333 + return nil, err 334 + } 335 + case <-wait: 336 + } 337 + 338 + e.l.Info("waited for container", "name", containerID) 339 + 340 + info, err := e.docker.ContainerInspect(ctx, containerID) 341 + if err != nil { 342 + return nil, err 343 + } 344 + 345 + return info.State, nil 346 + } 347 + 348 + func (e *Engine) tailStep(ctx context.Context, wfLogger *models.WorkflowLogger, containerID string, wid models.WorkflowId, stepIdx int, step models.Step) error { 349 + if wfLogger == nil { 350 + return nil 351 + } 352 + 353 + logs, err := e.docker.ContainerLogs(ctx, containerID, container.LogsOptions{ 354 + Follow: true, 355 + ShowStdout: true, 356 + ShowStderr: true, 357 + Details: false, 358 + Timestamps: false, 359 + }) 360 + if err != nil { 361 + return err 362 + } 363 + 364 + _, err = stdcopy.StdCopy( 365 + wfLogger.DataWriter("stdout"), 366 + wfLogger.DataWriter("stderr"), 367 + logs, 368 + ) 369 + if err != nil && err != io.EOF && !errors.Is(err, context.DeadlineExceeded) { 370 + return fmt.Errorf("failed to copy logs: %w", err) 371 + } 372 + 373 + return nil 374 + } 375 + 376 + func (e *Engine) DestroyStep(ctx context.Context, containerID string) error { 377 + err := e.docker.ContainerKill(ctx, containerID, "9") // SIGKILL 378 + if err != nil && !isErrContainerNotFoundOrNotRunning(err) { 379 + return err 380 + } 381 + 382 + if err := e.docker.ContainerRemove(ctx, containerID, container.RemoveOptions{ 383 + RemoveVolumes: true, 384 + RemoveLinks: false, 385 + Force: false, 386 + }); err != nil && !isErrContainerNotFoundOrNotRunning(err) { 387 + return err 388 + } 389 + 390 + return nil 391 + } 392 + 393 + func (e *Engine) DestroyWorkflow(ctx context.Context, wid models.WorkflowId) error { 394 + e.cleanupMu.Lock() 395 + key := wid.String() 396 + 397 + fns := e.cleanup[key] 398 + delete(e.cleanup, key) 399 + e.cleanupMu.Unlock() 400 + 401 + for _, fn := range fns { 402 + if err := fn(ctx); err != nil { 403 + e.l.Error("failed to cleanup workflow resource", "workflowId", wid, "error", err) 404 + } 405 + } 406 + return nil 407 + } 408 + 409 + func (e *Engine) registerCleanup(wid models.WorkflowId, fn cleanupFunc) { 410 + e.cleanupMu.Lock() 411 + defer e.cleanupMu.Unlock() 412 + 413 + key := wid.String() 414 + e.cleanup[key] = append(e.cleanup[key], fn) 415 + } 416 + 417 + func workspaceVolume(wid models.WorkflowId) string { 418 + return fmt.Sprintf("workspace-%s", wid) 419 + } 420 + 421 + func nixVolume(wid models.WorkflowId) string { 422 + return fmt.Sprintf("nix-%s", wid) 423 + } 424 + 425 + func networkName(wid models.WorkflowId) string { 426 + return fmt.Sprintf("workflow-network-%s", wid) 427 + } 428 + 429 + func hostConfig(wid models.WorkflowId) *container.HostConfig { 430 + hostConfig := &container.HostConfig{ 431 + Mounts: []mount.Mount{ 432 + { 433 + Type: mount.TypeVolume, 434 + Source: workspaceVolume(wid), 435 + Target: workspaceDir, 436 + }, 437 + { 438 + Type: mount.TypeVolume, 439 + Source: nixVolume(wid), 440 + Target: "/nix", 441 + }, 442 + { 443 + Type: mount.TypeTmpfs, 444 + Target: "/tmp", 445 + ReadOnly: false, 446 + TmpfsOptions: &mount.TmpfsOptions{ 447 + Mode: 0o1777, // world-writeable sticky bit 448 + Options: [][]string{ 449 + {"exec"}, 450 + }, 451 + }, 452 + }, 453 + { 454 + Type: mount.TypeVolume, 455 + Source: "etc-nix-" + wid.String(), 456 + Target: "/etc/nix", 457 + }, 458 + }, 459 + ReadonlyRootfs: false, 460 + CapDrop: []string{"ALL"}, 461 + CapAdd: []string{"CAP_DAC_OVERRIDE"}, 462 + SecurityOpt: []string{"no-new-privileges"}, 463 + ExtraHosts: []string{"host.docker.internal:host-gateway"}, 464 + } 465 + 466 + return hostConfig 467 + } 468 + 469 + // thanks woodpecker 470 + func isErrContainerNotFoundOrNotRunning(err error) bool { 471 + // Error response from daemon: Cannot kill container: ...: No such container: ... 472 + // Error response from daemon: Cannot kill container: ...: Container ... is not running" 473 + // Error response from podman daemon: can only kill running containers. ... is in state exited 474 + // Error: No such container: ... 475 + return err != nil && (strings.Contains(err.Error(), "No such container") || strings.Contains(err.Error(), "is not running") || strings.Contains(err.Error(), "can only kill running containers")) 476 + }

+7

spindle/engines/nixery/errors.go

··· 0 0 0 0 0 0 0

··· 1 + package nixery 2 + 3 + import "errors" 4 + 5 + var ( 6 + ErrOOMKilled = errors.New("oom killed") 7 + )

+17

spindle/models/engine.go

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

··· 1 + package models 2 + 3 + import ( 4 + "context" 5 + "time" 6 + 7 + "tangled.sh/tangled.sh/core/api/tangled" 8 + "tangled.sh/tangled.sh/core/spindle/secrets" 9 + ) 10 + 11 + type Engine interface { 12 + InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipeline) (*Workflow, error) 13 + SetupWorkflow(ctx context.Context, wid WorkflowId, wf *Workflow) error 14 + WorkflowTimeout() time.Duration 15 + DestroyWorkflow(ctx context.Context, wid WorkflowId) error 16 + RunStep(ctx context.Context, wid WorkflowId, w *Workflow, idx int, secrets []secrets.UnlockedSecret, wfLogger *WorkflowLogger) error 17 + }

+3 -3

spindle/models/models.go

··· 104 func NewControlLogLine(idx int, step Step) LogLine { 105 return LogLine{ 106 Kind: LogKindControl, 107 - Content: step.Name, 108 StepId: idx, 109 - StepKind: step.Kind, 110 - StepCommand: step.Command, 111 } 112 }

··· 104 func NewControlLogLine(idx int, step Step) LogLine { 105 return LogLine{ 106 Kind: LogKindControl, 107 + Content: step.Name(), 108 StepId: idx, 109 + StepKind: step.Kind(), 110 + StepCommand: step.Command(), 111 } 112 }

+8 -103

spindle/models/pipeline.go

··· 1 package models 2 3 - import ( 4 - "path" 5 - 6 - "tangled.sh/tangled.sh/core/api/tangled" 7 - "tangled.sh/tangled.sh/core/spindle/config" 8 - ) 9 - 10 type Pipeline struct { 11 RepoOwner string 12 RepoName string 13 - Workflows []Workflow 14 } 15 16 - type Step struct { 17 - Command string 18 - Name string 19 - Environment map[string]string 20 - Kind StepKind 21 } 22 23 type StepKind int ··· 30 ) 31 32 type Workflow struct { 33 - Steps []Step 34 - Environment map[string]string 35 - Name string 36 - Image string 37 - } 38 - 39 - // setupSteps get added to start of Steps 40 - type setupSteps []Step 41 - 42 - // addStep adds a step to the beginning of the workflow's steps. 43 - func (ss *setupSteps) addStep(step Step) { 44 - *ss = append(*ss, step) 45 - } 46 - 47 - // ToPipeline converts a tangled.Pipeline into a model.Pipeline. 48 - // In the process, dependencies are resolved: nixpkgs deps 49 - // are constructed atop nixery and set as the Workflow.Image, 50 - // and ones from custom registries 51 - func ToPipeline(pl tangled.Pipeline, cfg config.Config) *Pipeline { 52 - workflows := []Workflow{} 53 - 54 - for _, twf := range pl.Workflows { 55 - swf := &Workflow{} 56 - for _, tstep := range twf.Steps { 57 - sstep := Step{} 58 - sstep.Environment = stepEnvToMap(tstep.Environment) 59 - sstep.Command = tstep.Command 60 - sstep.Name = tstep.Name 61 - sstep.Kind = StepKindUser 62 - swf.Steps = append(swf.Steps, sstep) 63 - } 64 - swf.Name = twf.Name 65 - swf.Environment = workflowEnvToMap(twf.Environment) 66 - swf.Image = workflowImage(twf.Dependencies, cfg.Pipelines.Nixery) 67 - 68 - setup := &setupSteps{} 69 - 70 - setup.addStep(nixConfStep()) 71 - setup.addStep(cloneStep(*twf, *pl.TriggerMetadata, cfg.Server.Dev)) 72 - // this step could be empty 73 - if s := dependencyStep(*twf); s != nil { 74 - setup.addStep(*s) 75 - } 76 - 77 - // append setup steps in order to the start of workflow steps 78 - swf.Steps = append(*setup, swf.Steps...) 79 - 80 - workflows = append(workflows, *swf) 81 - } 82 - repoOwner := pl.TriggerMetadata.Repo.Did 83 - repoName := pl.TriggerMetadata.Repo.Repo 84 - return &Pipeline{ 85 - RepoOwner: repoOwner, 86 - RepoName: repoName, 87 - Workflows: workflows, 88 - } 89 - } 90 - 91 - func workflowEnvToMap(envs []*tangled.Pipeline_Pair) map[string]string { 92 - envMap := map[string]string{} 93 - for _, env := range envs { 94 - if env != nil { 95 - envMap[env.Key] = env.Value 96 - } 97 - } 98 - return envMap 99 - } 100 - 101 - func stepEnvToMap(envs []*tangled.Pipeline_Pair) map[string]string { 102 - envMap := map[string]string{} 103 - for _, env := range envs { 104 - if env != nil { 105 - envMap[env.Key] = env.Value 106 - } 107 - } 108 - return envMap 109 - } 110 - 111 - func workflowImage(deps []*tangled.Pipeline_Dependency, nixery string) string { 112 - var dependencies string 113 - for _, d := range deps { 114 - if d.Registry == "nixpkgs" { 115 - dependencies = path.Join(d.Packages...) 116 - } 117 - } 118 - 119 - // load defaults from somewhere else 120 - dependencies = path.Join(dependencies, "bash", "git", "coreutils", "nix") 121 - 122 - return path.Join(nixery, dependencies) 123 }

··· 1 package models 2 0 0 0 0 0 0 0 3 type Pipeline struct { 4 RepoOwner string 5 RepoName string 6 + Workflows map[Engine][]Workflow 7 } 8 9 + type Step interface { 10 + Name() string 11 + Command() string 12 + Kind() StepKind 0 13 } 14 15 type StepKind int ··· 22 ) 23 24 type Workflow struct { 25 + Steps []Step 26 + Name string 27 + Data any 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 28 }

+10 -13

spindle/models/setup_steps.go spindle/engines/nixery/setup_steps.go

··· 1 - package models 2 3 import ( 4 "fmt" ··· 13 setupCmd := `echo 'extra-experimental-features = nix-command flakes' >> /etc/nix/nix.conf 14 echo 'build-users-group = ' >> /etc/nix/nix.conf` 15 return Step{ 16 - Command: setupCmd, 17 - Name: "Configure Nix", 18 } 19 } 20 ··· 81 commands = append(commands, "git checkout FETCH_HEAD") 82 83 cloneStep := Step{ 84 - Command: strings.Join(commands, "\n"), 85 - Name: "Clone repository into workspace", 86 } 87 return cloneStep 88 } ··· 91 // For dependencies using a custom registry (i.e. not nixpkgs), it collects 92 // all packages and adds a single 'nix profile install' step to the 93 // beginning of the workflow's step list. 94 - func dependencyStep(twf tangled.Pipeline_Workflow) *Step { 95 var customPackages []string 96 97 - for _, d := range twf.Dependencies { 98 - registry := d.Registry 99 - packages := d.Packages 100 - 101 if registry == "nixpkgs" { 102 continue 103 } ··· 115 installCmd := "nix --extra-experimental-features nix-command --extra-experimental-features flakes profile install" 116 cmd := fmt.Sprintf("%s %s", installCmd, strings.Join(customPackages, " ")) 117 installStep := Step{ 118 - Command: cmd, 119 - Name: "Install custom dependencies", 120 - Environment: map[string]string{ 121 "NIX_NO_COLOR": "1", 122 "NIX_SHOW_DOWNLOAD_PROGRESS": "0", 123 },

··· 1 + package nixery 2 3 import ( 4 "fmt" ··· 13 setupCmd := `echo 'extra-experimental-features = nix-command flakes' >> /etc/nix/nix.conf 14 echo 'build-users-group = ' >> /etc/nix/nix.conf` 15 return Step{ 16 + command: setupCmd, 17 + name: "Configure Nix", 18 } 19 } 20 ··· 81 commands = append(commands, "git checkout FETCH_HEAD") 82 83 cloneStep := Step{ 84 + command: strings.Join(commands, "\n"), 85 + name: "Clone repository into workspace", 86 } 87 return cloneStep 88 } ··· 91 // For dependencies using a custom registry (i.e. not nixpkgs), it collects 92 // all packages and adds a single 'nix profile install' step to the 93 // beginning of the workflow's step list. 94 + func dependencyStep(deps map[string][]string) *Step { 95 var customPackages []string 96 97 + for registry, packages := range deps { 0 0 0 98 if registry == "nixpkgs" { 99 continue 100 } ··· 112 installCmd := "nix --extra-experimental-features nix-command --extra-experimental-features flakes profile install" 113 cmd := fmt.Sprintf("%s %s", installCmd, strings.Join(customPackages, " ")) 114 installStep := Step{ 115 + command: cmd, 116 + name: "Install custom dependencies", 117 + environment: map[string]string{ 118 "NIX_NO_COLOR": "1", 119 "NIX_SHOW_DOWNLOAD_PROGRESS": "0", 120 },

+38 -8

spindle/server.go

··· 20 "tangled.sh/tangled.sh/core/spindle/config" 21 "tangled.sh/tangled.sh/core/spindle/db" 22 "tangled.sh/tangled.sh/core/spindle/engine" 0 23 "tangled.sh/tangled.sh/core/spindle/models" 24 "tangled.sh/tangled.sh/core/spindle/queue" 25 "tangled.sh/tangled.sh/core/spindle/secrets" ··· 39 e *rbac.Enforcer 40 l *slog.Logger 41 n *notifier.Notifier 42 - eng *engine.Engine 43 jq *queue.Queue 44 cfg *config.Config 45 ks *eventconsumer.Consumer ··· 93 return fmt.Errorf("unknown secrets provider: %s", cfg.Server.Secrets.Provider) 94 } 95 96 - eng, err := engine.New(ctx, cfg, d, &n, vault) 97 if err != nil { 98 return err 99 } ··· 128 db: d, 129 l: logger, 130 n: &n, 131 - eng: eng, 132 jq: jq, 133 cfg: cfg, 134 res: resolver, ··· 216 Logger: logger, 217 Db: s.db, 218 Enforcer: s.e, 219 - Engine: s.eng, 220 Config: s.cfg, 221 Resolver: s.res, 222 Vault: s.vault, ··· 261 Rkey: msg.Rkey, 262 } 263 0 0 264 for _, w := range tpl.Workflows { 265 if w != nil { 266 - err := s.db.StatusPending(models.WorkflowId{ 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 267 PipelineId: pipelineId, 268 Name: w.Name, 269 }, s.n) ··· 273 } 274 } 275 276 - spl := models.ToPipeline(tpl, *s.cfg) 277 - 278 ok := s.jq.Enqueue(queue.Job{ 279 Run: func() error { 280 - s.eng.StartWorkflows(ctx, spl, pipelineId) 0 0 0 0 281 return nil 282 }, 283 OnFail: func(jobError error) {

··· 20 "tangled.sh/tangled.sh/core/spindle/config" 21 "tangled.sh/tangled.sh/core/spindle/db" 22 "tangled.sh/tangled.sh/core/spindle/engine" 23 + "tangled.sh/tangled.sh/core/spindle/engines/nixery" 24 "tangled.sh/tangled.sh/core/spindle/models" 25 "tangled.sh/tangled.sh/core/spindle/queue" 26 "tangled.sh/tangled.sh/core/spindle/secrets" ··· 40 e *rbac.Enforcer 41 l *slog.Logger 42 n *notifier.Notifier 43 + engs map[string]models.Engine 44 jq *queue.Queue 45 cfg *config.Config 46 ks *eventconsumer.Consumer ··· 94 return fmt.Errorf("unknown secrets provider: %s", cfg.Server.Secrets.Provider) 95 } 96 97 + nixeryEng, err := nixery.New(ctx, cfg) 98 if err != nil { 99 return err 100 } ··· 129 db: d, 130 l: logger, 131 n: &n, 132 + engs: map[string]models.Engine{"nixery": nixeryEng}, 133 jq: jq, 134 cfg: cfg, 135 res: resolver, ··· 217 Logger: logger, 218 Db: s.db, 219 Enforcer: s.e, 220 + Engines: s.engs, 221 Config: s.cfg, 222 Resolver: s.res, 223 Vault: s.vault, ··· 262 Rkey: msg.Rkey, 263 } 264 265 + workflows := make(map[models.Engine][]models.Workflow) 266 + 267 for _, w := range tpl.Workflows { 268 if w != nil { 269 + if _, ok := s.engs[w.Engine]; !ok { 270 + err = s.db.StatusFailed(models.WorkflowId{ 271 + PipelineId: pipelineId, 272 + Name: w.Name, 273 + }, fmt.Sprintf("unknown engine %#v", w.Engine), -1, s.n) 274 + if err != nil { 275 + return err 276 + } 277 + 278 + continue 279 + } 280 + 281 + eng := s.engs[w.Engine] 282 + 283 + if _, ok := workflows[eng]; !ok { 284 + workflows[eng] = []models.Workflow{} 285 + } 286 + 287 + ewf, err := s.engs[w.Engine].InitWorkflow(*w, tpl) 288 + if err != nil { 289 + return err 290 + } 291 + 292 + workflows[eng] = append(workflows[eng], *ewf) 293 + 294 + err = s.db.StatusPending(models.WorkflowId{ 295 PipelineId: pipelineId, 296 Name: w.Name, 297 }, s.n) ··· 301 } 302 } 303 0 0 304 ok := s.jq.Enqueue(queue.Job{ 305 Run: func() error { 306 + engine.StartWorkflows(s.l, s.vault, s.cfg, s.db, s.n, ctx, &models.Pipeline{ 307 + RepoOwner: tpl.TriggerMetadata.Repo.Did, 308 + RepoName: tpl.TriggerMetadata.Repo.Repo, 309 + Workflows: workflows, 310 + }, pipelineId) 311 return nil 312 }, 313 OnFail: func(jobError error) {

+1 -2

spindle/stream.go

··· 9 "strconv" 10 "time" 11 12 - "tangled.sh/tangled.sh/core/spindle/engine" 13 "tangled.sh/tangled.sh/core/spindle/models" 14 15 "github.com/go-chi/chi/v5" ··· 143 } 144 isFinished := models.StatusKind(status.Status).IsFinish() 145 146 - filePath := engine.LogFilePath(s.cfg.Pipelines.LogDir, wid) 147 148 config := tail.Config{ 149 Follow: !isFinished,

··· 9 "strconv" 10 "time" 11 0 12 "tangled.sh/tangled.sh/core/spindle/models" 13 14 "github.com/go-chi/chi/v5" ··· 142 } 143 isFinished := models.StatusKind(status.Status).IsFinish() 144 145 + filePath := models.LogFilePath(s.cfg.Server.LogDir, wid) 146 147 config := tail.Config{ 148 Follow: !isFinished,

+2 -2

spindle/xrpc/xrpc.go

··· 17 "tangled.sh/tangled.sh/core/rbac" 18 "tangled.sh/tangled.sh/core/spindle/config" 19 "tangled.sh/tangled.sh/core/spindle/db" 20 - "tangled.sh/tangled.sh/core/spindle/engine" 21 "tangled.sh/tangled.sh/core/spindle/secrets" 22 ) 23 ··· 27 Logger *slog.Logger 28 Db *db.DB 29 Enforcer *rbac.Enforcer 30 - Engine *engine.Engine 31 Config *config.Config 32 Resolver *idresolver.Resolver 33 Vault secrets.Manager

··· 17 "tangled.sh/tangled.sh/core/rbac" 18 "tangled.sh/tangled.sh/core/spindle/config" 19 "tangled.sh/tangled.sh/core/spindle/db" 20 + "tangled.sh/tangled.sh/core/spindle/models" 21 "tangled.sh/tangled.sh/core/spindle/secrets" 22 ) 23 ··· 27 Logger *slog.Logger 28 Db *db.DB 29 Enforcer *rbac.Enforcer 30 + Engines map[string]models.Engine 31 Config *config.Config 32 Resolver *idresolver.Resolver 33 Vault secrets.Manager

+17 -36

workflow/compile.go

··· 1 package workflow 2 3 import ( 0 4 "fmt" 5 6 "tangled.sh/tangled.sh/core/api/tangled" ··· 63 return fmt.Sprintf("warning: %s: %s: %s", w.Path, w.Type, w.Reason) 64 } 65 0 0 0 0 66 type WarningKind string 67 68 var ( ··· 95 for _, wf := range p { 96 cw := compiler.compileWorkflow(wf) 97 98 - // empty workflows are not added to the pipeline 99 - if len(cw.Steps) == 0 { 100 continue 101 } 102 103 - cp.Workflows = append(cp.Workflows, &cw) 104 } 105 106 return cp 107 } 108 109 - func (compiler *Compiler) compileWorkflow(w Workflow) tangled.Pipeline_Workflow { 110 - cw := tangled.Pipeline_Workflow{} 111 112 if !w.Match(compiler.Trigger) { 113 compiler.Diagnostics.AddWarning( ··· 115 WorkflowSkipped, 116 fmt.Sprintf("did not match trigger %s", compiler.Trigger.Kind), 117 ) 118 - return cw 119 - } 120 - 121 - if len(w.Steps) == 0 { 122 - compiler.Diagnostics.AddWarning( 123 - w.Name, 124 - WorkflowSkipped, 125 - "empty workflow", 126 - ) 127 - return cw 128 } 129 130 // validate clone options 131 compiler.analyzeCloneOptions(w) 132 133 cw.Name = w.Name 134 - cw.Dependencies = w.Dependencies.AsRecord() 135 - for _, s := range w.Steps { 136 - step := tangled.Pipeline_Step{ 137 - Command: s.Command, 138 - Name: s.Name, 139 - } 140 - for k, v := range s.Environment { 141 - e := &tangled.Pipeline_Pair{ 142 - Key: k, 143 - Value: v, 144 - } 145 - step.Environment = append(step.Environment, e) 146 - } 147 - cw.Steps = append(cw.Steps, &step) 148 } 149 - for k, v := range w.Environment { 150 - e := &tangled.Pipeline_Pair{ 151 - Key: k, 152 - Value: v, 153 - } 154 - cw.Environment = append(cw.Environment, e) 155 - } 156 157 o := w.CloneOpts.AsRecord() 158 cw.Clone = &o

··· 1 package workflow 2 3 import ( 4 + "errors" 5 "fmt" 6 7 "tangled.sh/tangled.sh/core/api/tangled" ··· 64 return fmt.Sprintf("warning: %s: %s: %s", w.Path, w.Type, w.Reason) 65 } 66 67 + var ( 68 + MissingEngine error = errors.New("missing engine") 69 + ) 70 + 71 type WarningKind string 72 73 var ( ··· 100 for _, wf := range p { 101 cw := compiler.compileWorkflow(wf) 102 103 + if cw == nil { 0 104 continue 105 } 106 107 + cp.Workflows = append(cp.Workflows, cw) 108 } 109 110 return cp 111 } 112 113 + func (compiler *Compiler) compileWorkflow(w Workflow) *tangled.Pipeline_Workflow { 114 + cw := &tangled.Pipeline_Workflow{} 115 116 if !w.Match(compiler.Trigger) { 117 compiler.Diagnostics.AddWarning( ··· 119 WorkflowSkipped, 120 fmt.Sprintf("did not match trigger %s", compiler.Trigger.Kind), 121 ) 122 + return nil 0 0 0 0 0 0 0 0 0 123 } 124 125 // validate clone options 126 compiler.analyzeCloneOptions(w) 127 128 cw.Name = w.Name 129 + 130 + if w.Engine == "" { 131 + compiler.Diagnostics.AddError(w.Name, MissingEngine) 132 + return nil 0 0 0 0 0 0 0 0 0 0 133 } 134 + 135 + cw.Engine = w.Engine 136 + cw.Raw = w.Raw 0 0 0 0 137 138 o := w.CloneOpts.AsRecord() 139 cw.Clone = &o

+23 -29

workflow/compile_test.go

··· 26 27 func TestCompileWorkflow_MatchingWorkflowWithSteps(t *testing.T) { 28 wf := Workflow{ 29 - Name: ".tangled/workflows/test.yml", 30 - When: when, 31 - Steps: []Step{ 32 - {Name: "Test", Command: "go test ./..."}, 33 - }, 34 CloneOpts: CloneOpts{}, // default true 35 } 36 ··· 43 assert.False(t, c.Diagnostics.IsErr()) 44 } 45 46 - func TestCompileWorkflow_EmptySteps(t *testing.T) { 47 - wf := Workflow{ 48 - Name: ".tangled/workflows/empty.yml", 49 - When: when, 50 - Steps: []Step{}, // no steps 51 - } 52 - 53 - c := Compiler{Trigger: trigger} 54 - cp := c.Compile([]Workflow{wf}) 55 - 56 - assert.Len(t, cp.Workflows, 0) 57 - assert.Len(t, c.Diagnostics.Warnings, 1) 58 - assert.Equal(t, WorkflowSkipped, c.Diagnostics.Warnings[0].Type) 59 - } 60 - 61 func TestCompileWorkflow_TriggerMismatch(t *testing.T) { 62 wf := Workflow{ 63 - Name: ".tangled/workflows/mismatch.yml", 0 64 When: []Constraint{ 65 { 66 Event: []string{"push"}, 67 Branch: []string{"master"}, // different branch 68 }, 69 }, 70 - Steps: []Step{ 71 - {Name: "Lint", Command: "golint ./..."}, 72 - }, 73 } 74 75 c := Compiler{Trigger: trigger} ··· 82 83 func TestCompileWorkflow_CloneFalseWithShallowTrue(t *testing.T) { 84 wf := Workflow{ 85 - Name: ".tangled/workflows/clone_skip.yml", 86 - When: when, 87 - Steps: []Step{ 88 - {Name: "Skip", Command: "echo skip"}, 89 - }, 90 CloneOpts: CloneOpts{ 91 Skip: true, 92 Depth: 1, ··· 101 assert.Len(t, c.Diagnostics.Warnings, 1) 102 assert.Equal(t, InvalidConfiguration, c.Diagnostics.Warnings[0].Type) 103 } 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

··· 26 27 func TestCompileWorkflow_MatchingWorkflowWithSteps(t *testing.T) { 28 wf := Workflow{ 29 + Name: ".tangled/workflows/test.yml", 30 + Engine: "nixery", 31 + When: when, 0 0 32 CloneOpts: CloneOpts{}, // default true 33 } 34 ··· 41 assert.False(t, c.Diagnostics.IsErr()) 42 } 43 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 44 func TestCompileWorkflow_TriggerMismatch(t *testing.T) { 45 wf := Workflow{ 46 + Name: ".tangled/workflows/mismatch.yml", 47 + Engine: "nixery", 48 When: []Constraint{ 49 { 50 Event: []string{"push"}, 51 Branch: []string{"master"}, // different branch 52 }, 53 }, 0 0 0 54 } 55 56 c := Compiler{Trigger: trigger} ··· 63 64 func TestCompileWorkflow_CloneFalseWithShallowTrue(t *testing.T) { 65 wf := Workflow{ 66 + Name: ".tangled/workflows/clone_skip.yml", 67 + Engine: "nixery", 68 + When: when, 0 0 69 CloneOpts: CloneOpts{ 70 Skip: true, 71 Depth: 1, ··· 80 assert.Len(t, c.Diagnostics.Warnings, 1) 81 assert.Equal(t, InvalidConfiguration, c.Diagnostics.Warnings[0].Type) 82 } 83 + 84 + func TestCompileWorkflow_MissingEngine(t *testing.T) { 85 + wf := Workflow{ 86 + Name: ".tangled/workflows/missing_engine.yml", 87 + When: when, 88 + Engine: "", 89 + } 90 + 91 + c := Compiler{Trigger: trigger} 92 + cp := c.Compile([]Workflow{wf}) 93 + 94 + assert.Len(t, cp.Workflows, 0) 95 + assert.Len(t, c.Diagnostics.Errors, 1) 96 + assert.Equal(t, MissingEngine, c.Diagnostics.Errors[0].Error) 97 + }

+6 -33

workflow/def.go

··· 24 25 // this is simply a structural representation of the workflow file 26 Workflow struct { 27 - Name string `yaml:"-"` // name of the workflow file 28 - When []Constraint `yaml:"when"` 29 - Dependencies Dependencies `yaml:"dependencies"` 30 - Steps []Step `yaml:"steps"` 31 - Environment map[string]string `yaml:"environment"` 32 - CloneOpts CloneOpts `yaml:"clone"` 33 } 34 35 Constraint struct { 36 Event StringList `yaml:"event"` 37 Branch StringList `yaml:"branch"` // this is optional, and only applied on "push" events 38 } 39 - 40 - Dependencies map[string][]string 41 42 CloneOpts struct { 43 Skip bool `yaml:"skip"` 44 Depth int `yaml:"depth"` 45 IncludeSubmodules bool `yaml:"submodules"` 46 - } 47 - 48 - Step struct { 49 - Name string `yaml:"name"` 50 - Command string `yaml:"command"` 51 - Environment map[string]string `yaml:"environment"` 52 } 53 54 StringList []string ··· 77 } 78 79 wf.Name = name 0 80 81 return wf, nil 82 } ··· 173 } 174 175 return errors.New("failed to unmarshal StringOrSlice") 176 - } 177 - 178 - // conversion utilities to atproto records 179 - func (d Dependencies) AsRecord() []*tangled.Pipeline_Dependency { 180 - var deps []*tangled.Pipeline_Dependency 181 - for registry, packages := range d { 182 - deps = append(deps, &tangled.Pipeline_Dependency{ 183 - Registry: registry, 184 - Packages: packages, 185 - }) 186 - } 187 - return deps 188 - } 189 - 190 - func (s Step) AsRecord() tangled.Pipeline_Step { 191 - return tangled.Pipeline_Step{ 192 - Command: s.Command, 193 - Name: s.Name, 194 - } 195 } 196 197 func (c CloneOpts) AsRecord() tangled.Pipeline_CloneOpts {

··· 24 25 // this is simply a structural representation of the workflow file 26 Workflow struct { 27 + Name string `yaml:"-"` // name of the workflow file 28 + Engine string `yaml:"engine"` 29 + When []Constraint `yaml:"when"` 30 + CloneOpts CloneOpts `yaml:"clone"` 31 + Raw string `yaml:"-"` 0 32 } 33 34 Constraint struct { 35 Event StringList `yaml:"event"` 36 Branch StringList `yaml:"branch"` // this is optional, and only applied on "push" events 37 } 0 0 38 39 CloneOpts struct { 40 Skip bool `yaml:"skip"` 41 Depth int `yaml:"depth"` 42 IncludeSubmodules bool `yaml:"submodules"` 0 0 0 0 0 0 43 } 44 45 StringList []string ··· 68 } 69 70 wf.Name = name 71 + wf.Raw = string(contents) 72 73 return wf, nil 74 } ··· 165 } 166 167 return errors.New("failed to unmarshal StringOrSlice") 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 168 } 169 170 func (c CloneOpts) AsRecord() tangled.Pipeline_CloneOpts {

+1 -86

workflow/def_test.go

··· 10 yamlData := ` 11 when: 12 - event: ["push", "pull_request"] 13 - branch: ["main", "develop"] 14 - 15 - dependencies: 16 - nixpkgs: 17 - - go 18 - - git 19 - - curl 20 - 21 - steps: 22 - - name: "Test" 23 - command: | 24 - go test ./...` 25 26 wf, err := FromFile("test.yml", []byte(yamlData)) 27 assert.NoError(t, err, "YAML should unmarshal without error") ··· 30 assert.ElementsMatch(t, []string{"main", "develop"}, wf.When[0].Branch) 31 assert.ElementsMatch(t, []string{"push", "pull_request"}, wf.When[0].Event) 32 33 - assert.Len(t, wf.Steps, 1) 34 - assert.Equal(t, "Test", wf.Steps[0].Name) 35 - assert.Equal(t, "go test ./...", wf.Steps[0].Command) 36 - 37 - pkgs, ok := wf.Dependencies["nixpkgs"] 38 - assert.True(t, ok, "`nixpkgs` should be present in dependencies") 39 - assert.ElementsMatch(t, []string{"go", "git", "curl"}, pkgs) 40 - 41 assert.False(t, wf.CloneOpts.Skip, "Skip should default to false") 42 } 43 44 - func TestUnmarshalCustomRegistry(t *testing.T) { 45 - yamlData := ` 46 - when: 47 - - event: push 48 - branch: main 49 - 50 - dependencies: 51 - git+https://tangled.sh/@oppi.li/tbsp: 52 - - tbsp 53 - git+https://git.peppe.rs/languages/statix: 54 - - statix 55 - 56 - steps: 57 - - name: "Check" 58 - command: | 59 - statix check` 60 - 61 - wf, err := FromFile("test.yml", []byte(yamlData)) 62 - assert.NoError(t, err, "YAML should unmarshal without error") 63 - 64 - assert.ElementsMatch(t, []string{"push"}, wf.When[0].Event) 65 - assert.ElementsMatch(t, []string{"main"}, wf.When[0].Branch) 66 - 67 - assert.ElementsMatch(t, []string{"tbsp"}, wf.Dependencies["git+https://tangled.sh/@oppi.li/tbsp"]) 68 - assert.ElementsMatch(t, []string{"statix"}, wf.Dependencies["git+https://git.peppe.rs/languages/statix"]) 69 - } 70 - 71 func TestUnmarshalCloneFalse(t *testing.T) { 72 yamlData := ` 73 when: ··· 75 76 clone: 77 skip: true 78 - 79 - dependencies: 80 - nixpkgs: 81 - - python3 82 - 83 - steps: 84 - - name: Notify 85 - command: | 86 - python3 ./notify.py 87 ` 88 89 wf, err := FromFile("test.yml", []byte(yamlData)) ··· 93 94 assert.True(t, wf.CloneOpts.Skip, "Skip should be false") 95 } 96 - 97 - func TestUnmarshalEnv(t *testing.T) { 98 - yamlData := ` 99 - when: 100 - - event: ["pull_request_close"] 101 - 102 - clone: 103 - skip: false 104 - 105 - environment: 106 - HOME: /home/foo bar/baz 107 - CGO_ENABLED: 1 108 - 109 - steps: 110 - - name: Something 111 - command: echo "hello" 112 - environment: 113 - FOO: bar 114 - BAZ: qux 115 - ` 116 - 117 - wf, err := FromFile("test.yml", []byte(yamlData)) 118 - assert.NoError(t, err) 119 - 120 - assert.Len(t, wf.Environment, 2) 121 - assert.Equal(t, "/home/foo bar/baz", wf.Environment["HOME"]) 122 - assert.Equal(t, "1", wf.Environment["CGO_ENABLED"]) 123 - assert.Equal(t, "bar", wf.Steps[0].Environment["FOO"]) 124 - assert.Equal(t, "qux", wf.Steps[0].Environment["BAZ"]) 125 - }

··· 10 yamlData := ` 11 when: 12 - event: ["push", "pull_request"] 13 + branch: ["main", "develop"]` 0 0 0 0 0 0 0 0 0 0 0 14 15 wf, err := FromFile("test.yml", []byte(yamlData)) 16 assert.NoError(t, err, "YAML should unmarshal without error") ··· 19 assert.ElementsMatch(t, []string{"main", "develop"}, wf.When[0].Branch) 20 assert.ElementsMatch(t, []string{"push", "pull_request"}, wf.When[0].Event) 21 0 0 0 0 0 0 0 0 22 assert.False(t, wf.CloneOpts.Skip, "Skip should default to false") 23 } 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 25 func TestUnmarshalCloneFalse(t *testing.T) { 26 yamlData := ` 27 when: ··· 29 30 clone: 31 skip: true 0 0 0 0 0 0 0 0 0 32 ` 33 34 wf, err := FromFile("test.yml", []byte(yamlData)) ··· 38 39 assert.True(t, wf.CloneOpts.Skip, "Skip should be false") 40 } 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0