Enable to run build and invoke in background

Signed-off-by: Kohei Tokunaga <ktokunaga.mail@gmail.com>

util/ioset/ioset.go

@@ -0,0 +1,256 @@
+package ioset
+
+import (
+	"io"
+	"sync"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+// Pipe returns a pair of piped readers and writers collection.
+// They are useful for controlling stdio stream using Forwarder function.
+func Pipe() (In, Out) {
+	r1, w1 := io.Pipe()
+	r2, w2 := io.Pipe()
+	r3, w3 := io.Pipe()
+	return In{r1, w2, w3}, Out{w1, r2, r3}
+}
+
+type In struct {
+	Stdin  io.ReadCloser
+	Stdout io.WriteCloser
+	Stderr io.WriteCloser
+}
+
+func (s In) Close() (retErr error) {
+	if err := s.Stdin.Close(); err != nil {
+		retErr = err
+	}
+	if err := s.Stdout.Close(); err != nil {
+		retErr = err
+	}
+	if err := s.Stderr.Close(); err != nil {
+		retErr = err
+	}
+	return
+}
+
+type Out struct {
+	Stdin  io.WriteCloser
+	Stdout io.ReadCloser
+	Stderr io.ReadCloser
+}
+
+func (s Out) Close() (retErr error) {
+	if err := s.Stdin.Close(); err != nil {
+		retErr = err
+	}
+	if err := s.Stdout.Close(); err != nil {
+		retErr = err
+	}
+	if err := s.Stderr.Close(); err != nil {
+		retErr = err
+	}
+	return
+}
+
+// Forwarder forwards IO between readers and writers contained
+// in In and Out structs.
+// In and Out can be changed during forwarding using SetIn and SetOut methods.
+type Forwarder struct {
+	stdin  *SingleForwarder
+	stdout *SingleForwarder
+	stderr *SingleForwarder
+	mu     sync.Mutex
+
+	// PropagateStdinClose indicates whether EOF from Stdin of Out should be propagated.
+	// If this is true, EOF from Stdin (reader) of Out closes Stdin (writer) of In.
+	PropagateStdinClose bool
+}
+
+func NewForwarder() *Forwarder {
+	return &Forwarder{
+		stdin:               NewSingleForwarder(),
+		stdout:              NewSingleForwarder(),
+		stderr:              NewSingleForwarder(),
+		PropagateStdinClose: true,
+	}
+}
+
+func (f *Forwarder) Close() (retErr error) {
+	if err := f.stdin.Close(); err != nil {
+		retErr = err
+	}
+	if err := f.stdout.Close(); err != nil {
+		retErr = err
+	}
+	if err := f.stderr.Close(); err != nil {
+		retErr = err
+	}
+	return retErr
+}
+
+func (f *Forwarder) SetOut(out *Out) {
+	f.mu.Lock()
+	if out == nil {
+		f.stdin.SetWriter(nil, func() io.WriteCloser { return nil })
+		f.stdout.SetReader(nil)
+		f.stderr.SetReader(nil)
+	} else {
+		f.stdin.SetWriter(out.Stdin, func() io.WriteCloser {
+			if f.PropagateStdinClose {
+				out.Stdin.Close() // propagate EOF
+				logrus.Debug("forwarder: propagating stdin close")
+				return nil
+			}
+			return out.Stdin
+		})
+		f.stdout.SetReader(out.Stdout)
+		f.stderr.SetReader(out.Stderr)
+	}
+	f.mu.Unlock()
+}
+
+func (f *Forwarder) SetIn(in *In) {
+	f.mu.Lock()
+	if in == nil {
+		f.stdin.SetReader(nil)
+		f.stdout.SetWriter(nil, func() io.WriteCloser { return nil })
+		f.stderr.SetWriter(nil, func() io.WriteCloser { return nil })
+	} else {
+		f.stdin.SetReader(in.Stdin)
+		f.stdout.SetWriter(in.Stdout, func() io.WriteCloser {
+			return in.Stdout // continue write; TODO: make it configurable if needed
+		})
+		f.stderr.SetWriter(in.Stderr, func() io.WriteCloser {
+			return in.Stderr // continue write; TODO: make it configurable if needed
+		})
+	}
+	f.mu.Unlock()
+}
+
+// SingleForwarder forwards IO from a reader to a writer.
+// The reader and writer can be changed during forwarding
+// using SetReader and SetWriter methods.
+type SingleForwarder struct {
+	curR           io.ReadCloser // closed when set another reader
+	curRMu         sync.Mutex
+	curW           io.WriteCloser // closed when set another writer
+	curWEOFHandler func() io.WriteCloser
+	curWMu         sync.Mutex
+
+	updateRCh chan io.ReadCloser
+	doneCh    chan struct{}
+
+	closeOnce sync.Once
+}
+
+func NewSingleForwarder() *SingleForwarder {
+	f := &SingleForwarder{
+		updateRCh: make(chan io.ReadCloser),
+		doneCh:    make(chan struct{}),
+	}
+	go f.doForward()
+	return f
+}
+
+func (f *SingleForwarder) doForward() {
+	var r io.ReadCloser
+	for {
+		readerInvalid := false
+		if r != nil {
+			go func() {
+				buf := make([]byte, 4096)
+				for {
+					n, readErr := r.Read(buf)
+					if readErr != nil && !errors.Is(readErr, io.EOF) && !errors.Is(readErr, io.ErrClosedPipe) {
+						logrus.Debugf("single forwarder: reader error: %v", readErr)
+						return
+					}
+					f.curWMu.Lock()
+					w := f.curW
+					f.curWMu.Unlock()
+					if w != nil {
+						if _, err := w.Write(buf[:n]); err != nil && !errors.Is(err, io.ErrClosedPipe) {
+							logrus.Debugf("single forwarder: writer error: %v", err)
+						}
+					}
+					if readerInvalid {
+						return
+					}
+					if readErr != io.EOF {
+						continue
+					}
+
+					f.curWMu.Lock()
+					var newW io.WriteCloser
+					if f.curWEOFHandler != nil {
+						newW = f.curWEOFHandler()
+					}
+					f.curW = newW
+					f.curWMu.Unlock()
+					return
+				}
+			}()
+		}
+		select {
+		case newR := <-f.updateRCh:
+			f.curRMu.Lock()
+			if f.curR != nil {
+				f.curR.Close()
+			}
+			f.curR = newR
+			r = newR
+			readerInvalid = true
+			f.curRMu.Unlock()
+		case <-f.doneCh:
+			return
+		}
+	}
+}
+
+// Close closes the both of registered reader and writer and finishes the forwarder.
+func (f *SingleForwarder) Close() (retErr error) {
+	f.closeOnce.Do(func() {
+		f.curRMu.Lock()
+		r := f.curR
+		f.curR = nil
+		f.curRMu.Unlock()
+		if r != nil {
+			if err := r.Close(); err != nil {
+				retErr = err
+			}
+		}
+		// TODO: Wait until read data fully written to the current writer if needed.
+		f.curWMu.Lock()
+		w := f.curW
+		f.curW = nil
+		f.curWMu.Unlock()
+		if w != nil {
+			if err := w.Close(); err != nil {
+				retErr = err
+			}
+		}
+		close(f.doneCh)
+	})
+	return retErr
+}
+
+// SetWriter sets the specified writer as the forward destination.
+// If curWEOFHandler isn't nil, this will be called when the current reader returns EOF.
+func (f *SingleForwarder) SetWriter(w io.WriteCloser, curWEOFHandler func() io.WriteCloser) {
+	f.curWMu.Lock()
+	if f.curW != nil {
+		// close all stream on the current IO no to mix with the new IO
+		f.curW.Close()
+	}
+	f.curW = w
+	f.curWEOFHandler = curWEOFHandler
+	f.curWMu.Unlock()
+}
+
+// SetWriter sets the specified reader as the forward source.
+func (f *SingleForwarder) SetReader(r io.ReadCloser) {
+	f.updateRCh <- r
+}

util/ioset/mux.go

@@ -0,0 +1,236 @@
+package ioset
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"sync"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+type MuxOut struct {
+	Out
+	EnableHook  func()
+	DisableHook func()
+}
+
+// NewMuxIO forwards IO stream to/from "in" and "outs".
+// It toggles IO when it detects "C-a-c" key.
+// "outs" are closed automatically when "in" reaches EOF.
+// "in" doesn't closed automatically so the caller needs to explicitly close it.
+func NewMuxIO(in In, outs []MuxOut, initIdx int, toggleMessage func(prev int, res int) string) *MuxIO {
+	m := &MuxIO{
+		enabled:       make(map[int]struct{}),
+		in:            in,
+		outs:          outs,
+		closedCh:      make(chan struct{}),
+		toggleMessage: toggleMessage,
+	}
+	for i := range outs {
+		m.enabled[i] = struct{}{}
+	}
+	m.maxCur = len(outs)
+	m.cur = initIdx
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+	for i, o := range outs {
+		i, o := i, o
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if err := copyToFunc(o.Stdout, func() (io.Writer, error) {
+				if m.cur == i {
+					return in.Stdout, nil
+				}
+				return nil, nil
+			}); err != nil {
+				logrus.WithField("output index", i).WithError(err).Warnf("failed to write stdout")
+			}
+			if err := o.Stdout.Close(); err != nil {
+				logrus.WithField("output index", i).WithError(err).Warnf("failed to close stdout")
+			}
+		}()
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if err := copyToFunc(o.Stderr, func() (io.Writer, error) {
+				if m.cur == i {
+					return in.Stderr, nil
+				}
+				return nil, nil
+			}); err != nil {
+				logrus.WithField("output index", i).WithError(err).Warnf("failed to write stderr")
+			}
+			if err := o.Stderr.Close(); err != nil {
+				logrus.WithField("output index", i).WithError(err).Warnf("failed to close stderr")
+			}
+		}()
+	}
+	go func() {
+		errToggle := errors.Errorf("toggle IO")
+		for {
+			prevIsControlSequence := false
+			if err := copyToFunc(traceReader(in.Stdin, func(r rune) (bool, error) {
+				// Toggle IO when it detects C-a-c
+				// TODO: make it configurable if needed
+				if int(r) == 1 {
+					prevIsControlSequence = true
+					return false, nil
+				}
+				defer func() { prevIsControlSequence = false }()
+				if prevIsControlSequence {
+					if string(r) == "c" {
+						return false, errToggle
+					}
+				}
+				return true, nil
+			}), func() (io.Writer, error) {
+				mu.Lock()
+				o := outs[m.cur]
+				mu.Unlock()
+				return o.Stdin, nil
+			}); !errors.Is(err, errToggle) {
+				if err != nil {
+					logrus.WithError(err).Warnf("failed to read stdin")
+				}
+				break
+			}
+			m.toggleIO()
+		}
+
+		// propagate stdin EOF
+		for i, o := range outs {
+			if err := o.Stdin.Close(); err != nil {
+				logrus.WithError(err).Warnf("failed to close stdin of %d", i)
+			}
+		}
+		wg.Wait()
+		close(m.closedCh)
+	}()
+	return m
+}
+
+type MuxIO struct {
+	cur           int
+	maxCur        int
+	enabled       map[int]struct{}
+	mu            sync.Mutex
+	in            In
+	outs          []MuxOut
+	closedCh      chan struct{}
+	toggleMessage func(prev int, res int) string
+}
+
+func (m *MuxIO) waitClosed() {
+	<-m.closedCh
+}
+
+func (m *MuxIO) Enable(i int) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.enabled[i] = struct{}{}
+}
+
+func (m *MuxIO) Disable(i int) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if i == 0 {
+		return errors.Errorf("disabling 0th io is prohibited")
+	}
+	delete(m.enabled, i)
+	if m.cur == i {
+		m.toggleIO()
+	}
+	return nil
+}
+
+func (m *MuxIO) toggleIO() {
+	if m.outs[m.cur].DisableHook != nil {
+		m.outs[m.cur].DisableHook()
+	}
+	prev := m.cur
+	for {
+		if m.cur+1 >= m.maxCur {
+			m.cur = 0
+		} else {
+			m.cur++
+		}
+		if _, ok := m.enabled[m.cur]; !ok {
+			continue
+		}
+		break
+	}
+	res := m.cur
+	if m.outs[m.cur].EnableHook != nil {
+		m.outs[m.cur].EnableHook()
+	}
+	fmt.Fprint(m.in.Stdout, m.toggleMessage(prev, res))
+}
+
+func traceReader(r io.ReadCloser, f func(rune) (bool, error)) io.ReadCloser {
+	pr, pw := io.Pipe()
+	go func() {
+		br := bufio.NewReader(r)
+		for {
+			rn, _, err := br.ReadRune()
+			if err != nil {
+				if err == io.EOF {
+					pw.Close()
+					return
+				}
+				pw.CloseWithError(err)
+				return
+			}
+			if isWrite, err := f(rn); err != nil {
+				pw.CloseWithError(err)
+				return
+			} else if !isWrite {
+				continue
+			}
+			if _, err := pw.Write([]byte(string(rn))); err != nil {
+				pw.CloseWithError(err)
+				return
+			}
+		}
+	}()
+	return &readerWithClose{
+		Reader: pr,
+		closeFunc: func() error {
+			pr.Close()
+			return r.Close()
+		},
+	}
+}
+
+func copyToFunc(r io.Reader, wFunc func() (io.Writer, error)) error {
+	buf := make([]byte, 4096)
+	for {
+		n, readErr := r.Read(buf)
+		if readErr != nil && readErr != io.EOF {
+			return readErr
+		}
+		w, err := wFunc()
+		if err != nil {
+			return err
+		}
+		if w != nil {
+			if _, err := w.Write(buf[:n]); err != nil {
+				logrus.WithError(err).Debugf("failed to copy")
+			}
+		}
+		if readErr == io.EOF {
+			return nil
+		}
+	}
+}
+
+type readerWithClose struct {
+	io.Reader
+	closeFunc func() error
+}
+
+func (r *readerWithClose) Close() error {
+	return r.closeFunc()
+}

util/ioset/mux_test.go

@@ -0,0 +1,321 @@
+package ioset
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"regexp"
+	"strings"
+	"testing"
+
+	"golang.org/x/sync/errgroup"
+)
+
+// TestMuxIO tests MuxIO
+func TestMuxIO(t *testing.T) {
+	tests := []struct {
+		name       string
+		inputs     []instruction
+		initIdx    int
+		outputsNum int
+		wants      []string
+
+		// Everytime string is written to the mux stdin, the output end
+		// that received the string write backs to the string that is masked with
+		// its index number. This is useful to check if writeback is written from the
+		// expected output destination.
+		wantsMaskedOutput string
+	}{
+		{
+			name: "single output",
+			inputs: []instruction{
+				input("foo\nbar\n"),
+				toggle(),
+				input("1234"),
+				toggle(),
+				input("456"),
+			},
+			initIdx:           0,
+			outputsNum:        1,
+			wants:             []string{"foo\nbar\n1234456"},
+			wantsMaskedOutput: `^0+$`,
+		},
+		{
+			name: "multi output",
+			inputs: []instruction{
+				input("foo\nbar\n"),
+				toggle(),
+				input("12" + string([]rune{rune(1)}) + "34abc"),
+				toggle(),
+				input("456"),
+			},
+			initIdx:           0,
+			outputsNum:        3,
+			wants:             []string{"foo\nbar\n", "1234abc", "456"},
+			wantsMaskedOutput: `^0+1+2+$`,
+		},
+		{
+			name: "multi output with nonzero index",
+			inputs: []instruction{
+				input("foo\nbar\n"),
+				toggle(),
+				input("1234"),
+				toggle(),
+				input("456"),
+			},
+			initIdx:           1,
+			outputsNum:        3,
+			wants:             []string{"456", "foo\nbar\n", "1234"},
+			wantsMaskedOutput: `^1+2+0+$`,
+		},
+		{
+			name: "multi output many toggles",
+			inputs: []instruction{
+				input("foo\nbar\n"),
+				toggle(),
+				input("1234"),
+				toggle(),
+				toggle(),
+				input("456"),
+				toggle(),
+				input("%%%%"),
+				toggle(),
+				toggle(),
+				toggle(),
+				input("aaaa"),
+			},
+			initIdx:           0,
+			outputsNum:        3,
+			wants:             []string{"foo\nbar\n456", "1234%%%%aaaa", ""},
+			wantsMaskedOutput: `^0+1+0+1+$`,
+		},
+		{
+			name: "enable disable",
+			inputs: []instruction{
+				input("foo\nbar\n"),
+				toggle(),
+				input("1234"),
+				toggle(),
+				input("456"),
+				disable(2),
+				input("%%%%"),
+				enable(2),
+				toggle(),
+				toggle(),
+				input("aaa"),
+				disable(2),
+				disable(1),
+				input("1111"),
+				toggle(),
+				input("2222"),
+				toggle(),
+				input("3333"),
+			},
+			initIdx:           0,
+			outputsNum:        3,
+			wants:             []string{"foo\nbar\n%%%%111122223333", "1234", "456aaa"},
+			wantsMaskedOutput: `^0+1+2+0+2+0+$`,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			inBuf, end, in := newTestIn(t)
+			var outBufs []*outBuf
+			var outs []MuxOut
+			if tt.outputsNum != len(tt.wants) {
+				t.Fatalf("wants != outputsNum")
+			}
+			for i := 0; i < tt.outputsNum; i++ {
+				outBuf, out := newTestOut(t, i)
+				outBufs = append(outBufs, outBuf)
+				outs = append(outs, MuxOut{out, nil, nil})
+			}
+			mio := NewMuxIO(in, outs, tt.initIdx, func(prev int, res int) string { return "" })
+			for _, i := range tt.inputs {
+				// Add input to MuxIO
+				istr, writeback := i(mio)
+				if _, err := end.Stdin.Write([]byte(istr)); err != nil {
+					t.Fatalf("failed to write data to stdin: %v", err)
+				}
+
+				// Wait for writeback of this input
+				var eg errgroup.Group
+				eg.Go(func() error {
+					outbuf := make([]byte, len(writeback))
+					if _, err := io.ReadAtLeast(end.Stdout, outbuf, len(outbuf)); err != nil {
+						return err
+					}
+					return nil
+				})
+				eg.Go(func() error {
+					errbuf := make([]byte, len(writeback))
+					if _, err := io.ReadAtLeast(end.Stderr, errbuf, len(errbuf)); err != nil {
+						return err
+					}
+					return nil
+				})
+				if err := eg.Wait(); err != nil {
+					t.Fatalf("failed to wait for output: %v", err)
+				}
+			}
+
+			// Close stdin on this MuxIO
+			end.Stdin.Close()
+
+			// Wait for all output ends reach EOF
+			mio.waitClosed()
+
+			// Close stdout/stderr as well
+			in.Close()
+
+			// Check if each output end received expected string
+			<-inBuf.doneCh
+			for i, o := range outBufs {
+				<-o.doneCh
+				if o.stdin != tt.wants[i] {
+					t.Fatalf("output[%d]: got %q; wanted %q", i, o.stdin, tt.wants[i])
+				}
+			}
+
+			// Check if expected string is returned from expected outputs
+			if !regexp.MustCompile(tt.wantsMaskedOutput).MatchString(inBuf.stdout) {
+				t.Fatalf("stdout: got %q; wanted %q", inBuf.stdout, tt.wantsMaskedOutput)
+			}
+			if !regexp.MustCompile(tt.wantsMaskedOutput).MatchString(inBuf.stderr) {
+				t.Fatalf("stderr: got %q; wanted %q", inBuf.stderr, tt.wantsMaskedOutput)
+			}
+		})
+	}
+}
+
+type instruction func(m *MuxIO) (intput string, writeBackView string)
+
+func input(s string) instruction {
+	return func(m *MuxIO) (string, string) {
+		return s, strings.ReplaceAll(s, string([]rune{rune(1)}), "")
+	}
+}
+
+func toggle() instruction {
+	return func(m *MuxIO) (string, string) {
+		return string([]rune{rune(1)}) + "c", ""
+	}
+}
+
+func enable(i int) instruction {
+	return func(m *MuxIO) (string, string) {
+		m.Enable(i)
+		return "", ""
+	}
+}
+
+func disable(i int) instruction {
+	return func(m *MuxIO) (string, string) {
+		m.Disable(i)
+		return "", ""
+	}
+}
+
+type inBuf struct {
+	stdout string
+	stderr string
+	doneCh chan struct{}
+}
+
+func newTestIn(t *testing.T) (*inBuf, Out, In) {
+	ti := &inBuf{
+		doneCh: make(chan struct{}),
+	}
+	gotOutR, gotOutW := io.Pipe()
+	gotErrR, gotErrW := io.Pipe()
+	outR, outW := io.Pipe()
+	var eg errgroup.Group
+	eg.Go(func() error {
+		buf := new(bytes.Buffer)
+		if _, err := io.Copy(io.MultiWriter(gotOutW, buf), outR); err != nil {
+			return err
+		}
+		ti.stdout = buf.String()
+		return nil
+	})
+	errR, errW := io.Pipe()
+	eg.Go(func() error {
+		buf := new(bytes.Buffer)
+		if _, err := io.Copy(io.MultiWriter(gotErrW, buf), errR); err != nil {
+			return err
+		}
+		ti.stderr = buf.String()
+		return nil
+	})
+	go func() {
+		eg.Wait()
+		close(ti.doneCh)
+	}()
+	inR, inW := io.Pipe()
+	return ti, Out{Stdin: inW, Stdout: gotOutR, Stderr: gotErrR}, In{Stdin: inR, Stdout: outW, Stderr: errW}
+}
+
+type outBuf struct {
+	idx    int
+	stdin  string
+	doneCh chan struct{}
+}
+
+func newTestOut(t *testing.T, idx int) (*outBuf, Out) {
+	to := &outBuf{
+		idx:    idx,
+		doneCh: make(chan struct{}),
+	}
+	inR, inW := io.Pipe()
+	outR, outW := io.Pipe()
+	errR, errW := io.Pipe()
+	go func() {
+		defer inR.Close()
+		defer outW.Close()
+		defer errW.Close()
+		buf := new(bytes.Buffer)
+		mw := io.MultiWriter(buf,
+			writeMasked(outW, fmt.Sprintf("%d", to.idx)),
+			writeMasked(errW, fmt.Sprintf("%d", to.idx)),
+		)
+		if _, err := io.Copy(mw, inR); err != nil {
+			inR.CloseWithError(err)
+			outW.CloseWithError(err)
+			errW.CloseWithError(err)
+			return
+		}
+		to.stdin = string(buf.Bytes())
+		outW.Close()
+		errW.Close()
+		close(to.doneCh)
+	}()
+	return to, Out{Stdin: inW, Stdout: outR, Stderr: errR}
+}
+
+func writeMasked(w io.Writer, s string) io.Writer {
+	buf := make([]byte, 4096)
+	pr, pw := io.Pipe()
+	go func() {
+		for {
+			n, readErr := pr.Read(buf)
+			if readErr != nil && readErr != io.EOF {
+				pr.CloseWithError(readErr)
+				return
+			}
+			var masked string
+			for i := 0; i < n; i++ {
+				masked += s
+			}
+			if _, err := w.Write([]byte(masked)); err != nil {
+				pr.CloseWithError(err)
+				return
+			}
+			if readErr == io.EOF {
+				pr.Close()
+				return
+			}
+		}
+	}()
+	return pw
+}