Enable to run build and invoke in background

Signed-off-by: Kohei Tokunaga <ktokunaga.mail@gmail.com>
This commit is contained in:
Kohei Tokunaga
2022-08-29 14:14:14 +09:00
parent b1b4e64c97
commit a27b8395b1
80 changed files with 24862 additions and 678 deletions

256
util/ioset/ioset.go Normal file
View File

@@ -0,0 +1,256 @@
package ioset
import (
"io"
"sync"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
// Pipe returns a pair of piped readers and writers collection.
// They are useful for controlling stdio stream using Forwarder function.
func Pipe() (In, Out) {
r1, w1 := io.Pipe()
r2, w2 := io.Pipe()
r3, w3 := io.Pipe()
return In{r1, w2, w3}, Out{w1, r2, r3}
}
type In struct {
Stdin io.ReadCloser
Stdout io.WriteCloser
Stderr io.WriteCloser
}
func (s In) Close() (retErr error) {
if err := s.Stdin.Close(); err != nil {
retErr = err
}
if err := s.Stdout.Close(); err != nil {
retErr = err
}
if err := s.Stderr.Close(); err != nil {
retErr = err
}
return
}
type Out struct {
Stdin io.WriteCloser
Stdout io.ReadCloser
Stderr io.ReadCloser
}
func (s Out) Close() (retErr error) {
if err := s.Stdin.Close(); err != nil {
retErr = err
}
if err := s.Stdout.Close(); err != nil {
retErr = err
}
if err := s.Stderr.Close(); err != nil {
retErr = err
}
return
}
// Forwarder forwards IO between readers and writers contained
// in In and Out structs.
// In and Out can be changed during forwarding using SetIn and SetOut methods.
type Forwarder struct {
stdin *SingleForwarder
stdout *SingleForwarder
stderr *SingleForwarder
mu sync.Mutex
// PropagateStdinClose indicates whether EOF from Stdin of Out should be propagated.
// If this is true, EOF from Stdin (reader) of Out closes Stdin (writer) of In.
PropagateStdinClose bool
}
func NewForwarder() *Forwarder {
return &Forwarder{
stdin: NewSingleForwarder(),
stdout: NewSingleForwarder(),
stderr: NewSingleForwarder(),
PropagateStdinClose: true,
}
}
func (f *Forwarder) Close() (retErr error) {
if err := f.stdin.Close(); err != nil {
retErr = err
}
if err := f.stdout.Close(); err != nil {
retErr = err
}
if err := f.stderr.Close(); err != nil {
retErr = err
}
return retErr
}
func (f *Forwarder) SetOut(out *Out) {
f.mu.Lock()
if out == nil {
f.stdin.SetWriter(nil, func() io.WriteCloser { return nil })
f.stdout.SetReader(nil)
f.stderr.SetReader(nil)
} else {
f.stdin.SetWriter(out.Stdin, func() io.WriteCloser {
if f.PropagateStdinClose {
out.Stdin.Close() // propagate EOF
logrus.Debug("forwarder: propagating stdin close")
return nil
}
return out.Stdin
})
f.stdout.SetReader(out.Stdout)
f.stderr.SetReader(out.Stderr)
}
f.mu.Unlock()
}
func (f *Forwarder) SetIn(in *In) {
f.mu.Lock()
if in == nil {
f.stdin.SetReader(nil)
f.stdout.SetWriter(nil, func() io.WriteCloser { return nil })
f.stderr.SetWriter(nil, func() io.WriteCloser { return nil })
} else {
f.stdin.SetReader(in.Stdin)
f.stdout.SetWriter(in.Stdout, func() io.WriteCloser {
return in.Stdout // continue write; TODO: make it configurable if needed
})
f.stderr.SetWriter(in.Stderr, func() io.WriteCloser {
return in.Stderr // continue write; TODO: make it configurable if needed
})
}
f.mu.Unlock()
}
// SingleForwarder forwards IO from a reader to a writer.
// The reader and writer can be changed during forwarding
// using SetReader and SetWriter methods.
type SingleForwarder struct {
curR io.ReadCloser // closed when set another reader
curRMu sync.Mutex
curW io.WriteCloser // closed when set another writer
curWEOFHandler func() io.WriteCloser
curWMu sync.Mutex
updateRCh chan io.ReadCloser
doneCh chan struct{}
closeOnce sync.Once
}
func NewSingleForwarder() *SingleForwarder {
f := &SingleForwarder{
updateRCh: make(chan io.ReadCloser),
doneCh: make(chan struct{}),
}
go f.doForward()
return f
}
func (f *SingleForwarder) doForward() {
var r io.ReadCloser
for {
readerInvalid := false
if r != nil {
go func() {
buf := make([]byte, 4096)
for {
n, readErr := r.Read(buf)
if readErr != nil && !errors.Is(readErr, io.EOF) && !errors.Is(readErr, io.ErrClosedPipe) {
logrus.Debugf("single forwarder: reader error: %v", readErr)
return
}
f.curWMu.Lock()
w := f.curW
f.curWMu.Unlock()
if w != nil {
if _, err := w.Write(buf[:n]); err != nil && !errors.Is(err, io.ErrClosedPipe) {
logrus.Debugf("single forwarder: writer error: %v", err)
}
}
if readerInvalid {
return
}
if readErr != io.EOF {
continue
}
f.curWMu.Lock()
var newW io.WriteCloser
if f.curWEOFHandler != nil {
newW = f.curWEOFHandler()
}
f.curW = newW
f.curWMu.Unlock()
return
}
}()
}
select {
case newR := <-f.updateRCh:
f.curRMu.Lock()
if f.curR != nil {
f.curR.Close()
}
f.curR = newR
r = newR
readerInvalid = true
f.curRMu.Unlock()
case <-f.doneCh:
return
}
}
}
// Close closes the both of registered reader and writer and finishes the forwarder.
func (f *SingleForwarder) Close() (retErr error) {
f.closeOnce.Do(func() {
f.curRMu.Lock()
r := f.curR
f.curR = nil
f.curRMu.Unlock()
if r != nil {
if err := r.Close(); err != nil {
retErr = err
}
}
// TODO: Wait until read data fully written to the current writer if needed.
f.curWMu.Lock()
w := f.curW
f.curW = nil
f.curWMu.Unlock()
if w != nil {
if err := w.Close(); err != nil {
retErr = err
}
}
close(f.doneCh)
})
return retErr
}
// SetWriter sets the specified writer as the forward destination.
// If curWEOFHandler isn't nil, this will be called when the current reader returns EOF.
func (f *SingleForwarder) SetWriter(w io.WriteCloser, curWEOFHandler func() io.WriteCloser) {
f.curWMu.Lock()
if f.curW != nil {
// close all stream on the current IO no to mix with the new IO
f.curW.Close()
}
f.curW = w
f.curWEOFHandler = curWEOFHandler
f.curWMu.Unlock()
}
// SetWriter sets the specified reader as the forward source.
func (f *SingleForwarder) SetReader(r io.ReadCloser) {
f.updateRCh <- r
}

236
util/ioset/mux.go Normal file
View File

@@ -0,0 +1,236 @@
package ioset
import (
"bufio"
"fmt"
"io"
"sync"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
type MuxOut struct {
Out
EnableHook func()
DisableHook func()
}
// NewMuxIO forwards IO stream to/from "in" and "outs".
// It toggles IO when it detects "C-a-c" key.
// "outs" are closed automatically when "in" reaches EOF.
// "in" doesn't closed automatically so the caller needs to explicitly close it.
func NewMuxIO(in In, outs []MuxOut, initIdx int, toggleMessage func(prev int, res int) string) *MuxIO {
m := &MuxIO{
enabled: make(map[int]struct{}),
in: in,
outs: outs,
closedCh: make(chan struct{}),
toggleMessage: toggleMessage,
}
for i := range outs {
m.enabled[i] = struct{}{}
}
m.maxCur = len(outs)
m.cur = initIdx
var wg sync.WaitGroup
var mu sync.Mutex
for i, o := range outs {
i, o := i, o
wg.Add(1)
go func() {
defer wg.Done()
if err := copyToFunc(o.Stdout, func() (io.Writer, error) {
if m.cur == i {
return in.Stdout, nil
}
return nil, nil
}); err != nil {
logrus.WithField("output index", i).WithError(err).Warnf("failed to write stdout")
}
if err := o.Stdout.Close(); err != nil {
logrus.WithField("output index", i).WithError(err).Warnf("failed to close stdout")
}
}()
wg.Add(1)
go func() {
defer wg.Done()
if err := copyToFunc(o.Stderr, func() (io.Writer, error) {
if m.cur == i {
return in.Stderr, nil
}
return nil, nil
}); err != nil {
logrus.WithField("output index", i).WithError(err).Warnf("failed to write stderr")
}
if err := o.Stderr.Close(); err != nil {
logrus.WithField("output index", i).WithError(err).Warnf("failed to close stderr")
}
}()
}
go func() {
errToggle := errors.Errorf("toggle IO")
for {
prevIsControlSequence := false
if err := copyToFunc(traceReader(in.Stdin, func(r rune) (bool, error) {
// Toggle IO when it detects C-a-c
// TODO: make it configurable if needed
if int(r) == 1 {
prevIsControlSequence = true
return false, nil
}
defer func() { prevIsControlSequence = false }()
if prevIsControlSequence {
if string(r) == "c" {
return false, errToggle
}
}
return true, nil
}), func() (io.Writer, error) {
mu.Lock()
o := outs[m.cur]
mu.Unlock()
return o.Stdin, nil
}); !errors.Is(err, errToggle) {
if err != nil {
logrus.WithError(err).Warnf("failed to read stdin")
}
break
}
m.toggleIO()
}
// propagate stdin EOF
for i, o := range outs {
if err := o.Stdin.Close(); err != nil {
logrus.WithError(err).Warnf("failed to close stdin of %d", i)
}
}
wg.Wait()
close(m.closedCh)
}()
return m
}
type MuxIO struct {
cur int
maxCur int
enabled map[int]struct{}
mu sync.Mutex
in In
outs []MuxOut
closedCh chan struct{}
toggleMessage func(prev int, res int) string
}
func (m *MuxIO) waitClosed() {
<-m.closedCh
}
func (m *MuxIO) Enable(i int) {
m.mu.Lock()
defer m.mu.Unlock()
m.enabled[i] = struct{}{}
}
func (m *MuxIO) Disable(i int) error {
m.mu.Lock()
defer m.mu.Unlock()
if i == 0 {
return errors.Errorf("disabling 0th io is prohibited")
}
delete(m.enabled, i)
if m.cur == i {
m.toggleIO()
}
return nil
}
func (m *MuxIO) toggleIO() {
if m.outs[m.cur].DisableHook != nil {
m.outs[m.cur].DisableHook()
}
prev := m.cur
for {
if m.cur+1 >= m.maxCur {
m.cur = 0
} else {
m.cur++
}
if _, ok := m.enabled[m.cur]; !ok {
continue
}
break
}
res := m.cur
if m.outs[m.cur].EnableHook != nil {
m.outs[m.cur].EnableHook()
}
fmt.Fprint(m.in.Stdout, m.toggleMessage(prev, res))
}
func traceReader(r io.ReadCloser, f func(rune) (bool, error)) io.ReadCloser {
pr, pw := io.Pipe()
go func() {
br := bufio.NewReader(r)
for {
rn, _, err := br.ReadRune()
if err != nil {
if err == io.EOF {
pw.Close()
return
}
pw.CloseWithError(err)
return
}
if isWrite, err := f(rn); err != nil {
pw.CloseWithError(err)
return
} else if !isWrite {
continue
}
if _, err := pw.Write([]byte(string(rn))); err != nil {
pw.CloseWithError(err)
return
}
}
}()
return &readerWithClose{
Reader: pr,
closeFunc: func() error {
pr.Close()
return r.Close()
},
}
}
func copyToFunc(r io.Reader, wFunc func() (io.Writer, error)) error {
buf := make([]byte, 4096)
for {
n, readErr := r.Read(buf)
if readErr != nil && readErr != io.EOF {
return readErr
}
w, err := wFunc()
if err != nil {
return err
}
if w != nil {
if _, err := w.Write(buf[:n]); err != nil {
logrus.WithError(err).Debugf("failed to copy")
}
}
if readErr == io.EOF {
return nil
}
}
}
type readerWithClose struct {
io.Reader
closeFunc func() error
}
func (r *readerWithClose) Close() error {
return r.closeFunc()
}

321
util/ioset/mux_test.go Normal file
View File

@@ -0,0 +1,321 @@
package ioset
import (
"bytes"
"fmt"
"io"
"regexp"
"strings"
"testing"
"golang.org/x/sync/errgroup"
)
// TestMuxIO tests MuxIO
func TestMuxIO(t *testing.T) {
tests := []struct {
name string
inputs []instruction
initIdx int
outputsNum int
wants []string
// Everytime string is written to the mux stdin, the output end
// that received the string write backs to the string that is masked with
// its index number. This is useful to check if writeback is written from the
// expected output destination.
wantsMaskedOutput string
}{
{
name: "single output",
inputs: []instruction{
input("foo\nbar\n"),
toggle(),
input("1234"),
toggle(),
input("456"),
},
initIdx: 0,
outputsNum: 1,
wants: []string{"foo\nbar\n1234456"},
wantsMaskedOutput: `^0+$`,
},
{
name: "multi output",
inputs: []instruction{
input("foo\nbar\n"),
toggle(),
input("12" + string([]rune{rune(1)}) + "34abc"),
toggle(),
input("456"),
},
initIdx: 0,
outputsNum: 3,
wants: []string{"foo\nbar\n", "1234abc", "456"},
wantsMaskedOutput: `^0+1+2+$`,
},
{
name: "multi output with nonzero index",
inputs: []instruction{
input("foo\nbar\n"),
toggle(),
input("1234"),
toggle(),
input("456"),
},
initIdx: 1,
outputsNum: 3,
wants: []string{"456", "foo\nbar\n", "1234"},
wantsMaskedOutput: `^1+2+0+$`,
},
{
name: "multi output many toggles",
inputs: []instruction{
input("foo\nbar\n"),
toggle(),
input("1234"),
toggle(),
toggle(),
input("456"),
toggle(),
input("%%%%"),
toggle(),
toggle(),
toggle(),
input("aaaa"),
},
initIdx: 0,
outputsNum: 3,
wants: []string{"foo\nbar\n456", "1234%%%%aaaa", ""},
wantsMaskedOutput: `^0+1+0+1+$`,
},
{
name: "enable disable",
inputs: []instruction{
input("foo\nbar\n"),
toggle(),
input("1234"),
toggle(),
input("456"),
disable(2),
input("%%%%"),
enable(2),
toggle(),
toggle(),
input("aaa"),
disable(2),
disable(1),
input("1111"),
toggle(),
input("2222"),
toggle(),
input("3333"),
},
initIdx: 0,
outputsNum: 3,
wants: []string{"foo\nbar\n%%%%111122223333", "1234", "456aaa"},
wantsMaskedOutput: `^0+1+2+0+2+0+$`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
inBuf, end, in := newTestIn(t)
var outBufs []*outBuf
var outs []MuxOut
if tt.outputsNum != len(tt.wants) {
t.Fatalf("wants != outputsNum")
}
for i := 0; i < tt.outputsNum; i++ {
outBuf, out := newTestOut(t, i)
outBufs = append(outBufs, outBuf)
outs = append(outs, MuxOut{out, nil, nil})
}
mio := NewMuxIO(in, outs, tt.initIdx, func(prev int, res int) string { return "" })
for _, i := range tt.inputs {
// Add input to MuxIO
istr, writeback := i(mio)
if _, err := end.Stdin.Write([]byte(istr)); err != nil {
t.Fatalf("failed to write data to stdin: %v", err)
}
// Wait for writeback of this input
var eg errgroup.Group
eg.Go(func() error {
outbuf := make([]byte, len(writeback))
if _, err := io.ReadAtLeast(end.Stdout, outbuf, len(outbuf)); err != nil {
return err
}
return nil
})
eg.Go(func() error {
errbuf := make([]byte, len(writeback))
if _, err := io.ReadAtLeast(end.Stderr, errbuf, len(errbuf)); err != nil {
return err
}
return nil
})
if err := eg.Wait(); err != nil {
t.Fatalf("failed to wait for output: %v", err)
}
}
// Close stdin on this MuxIO
end.Stdin.Close()
// Wait for all output ends reach EOF
mio.waitClosed()
// Close stdout/stderr as well
in.Close()
// Check if each output end received expected string
<-inBuf.doneCh
for i, o := range outBufs {
<-o.doneCh
if o.stdin != tt.wants[i] {
t.Fatalf("output[%d]: got %q; wanted %q", i, o.stdin, tt.wants[i])
}
}
// Check if expected string is returned from expected outputs
if !regexp.MustCompile(tt.wantsMaskedOutput).MatchString(inBuf.stdout) {
t.Fatalf("stdout: got %q; wanted %q", inBuf.stdout, tt.wantsMaskedOutput)
}
if !regexp.MustCompile(tt.wantsMaskedOutput).MatchString(inBuf.stderr) {
t.Fatalf("stderr: got %q; wanted %q", inBuf.stderr, tt.wantsMaskedOutput)
}
})
}
}
type instruction func(m *MuxIO) (intput string, writeBackView string)
func input(s string) instruction {
return func(m *MuxIO) (string, string) {
return s, strings.ReplaceAll(s, string([]rune{rune(1)}), "")
}
}
func toggle() instruction {
return func(m *MuxIO) (string, string) {
return string([]rune{rune(1)}) + "c", ""
}
}
func enable(i int) instruction {
return func(m *MuxIO) (string, string) {
m.Enable(i)
return "", ""
}
}
func disable(i int) instruction {
return func(m *MuxIO) (string, string) {
m.Disable(i)
return "", ""
}
}
type inBuf struct {
stdout string
stderr string
doneCh chan struct{}
}
func newTestIn(t *testing.T) (*inBuf, Out, In) {
ti := &inBuf{
doneCh: make(chan struct{}),
}
gotOutR, gotOutW := io.Pipe()
gotErrR, gotErrW := io.Pipe()
outR, outW := io.Pipe()
var eg errgroup.Group
eg.Go(func() error {
buf := new(bytes.Buffer)
if _, err := io.Copy(io.MultiWriter(gotOutW, buf), outR); err != nil {
return err
}
ti.stdout = buf.String()
return nil
})
errR, errW := io.Pipe()
eg.Go(func() error {
buf := new(bytes.Buffer)
if _, err := io.Copy(io.MultiWriter(gotErrW, buf), errR); err != nil {
return err
}
ti.stderr = buf.String()
return nil
})
go func() {
eg.Wait()
close(ti.doneCh)
}()
inR, inW := io.Pipe()
return ti, Out{Stdin: inW, Stdout: gotOutR, Stderr: gotErrR}, In{Stdin: inR, Stdout: outW, Stderr: errW}
}
type outBuf struct {
idx int
stdin string
doneCh chan struct{}
}
func newTestOut(t *testing.T, idx int) (*outBuf, Out) {
to := &outBuf{
idx: idx,
doneCh: make(chan struct{}),
}
inR, inW := io.Pipe()
outR, outW := io.Pipe()
errR, errW := io.Pipe()
go func() {
defer inR.Close()
defer outW.Close()
defer errW.Close()
buf := new(bytes.Buffer)
mw := io.MultiWriter(buf,
writeMasked(outW, fmt.Sprintf("%d", to.idx)),
writeMasked(errW, fmt.Sprintf("%d", to.idx)),
)
if _, err := io.Copy(mw, inR); err != nil {
inR.CloseWithError(err)
outW.CloseWithError(err)
errW.CloseWithError(err)
return
}
to.stdin = string(buf.Bytes())
outW.Close()
errW.Close()
close(to.doneCh)
}()
return to, Out{Stdin: inW, Stdout: outR, Stderr: errR}
}
func writeMasked(w io.Writer, s string) io.Writer {
buf := make([]byte, 4096)
pr, pw := io.Pipe()
go func() {
for {
n, readErr := pr.Read(buf)
if readErr != nil && readErr != io.EOF {
pr.CloseWithError(readErr)
return
}
var masked string
for i := 0; i < n; i++ {
masked += s
}
if _, err := w.Write([]byte(masked)); err != nil {
pr.CloseWithError(err)
return
}
if readErr == io.EOF {
pr.Close()
return
}
}
}()
return pw
}