diff --git a/bake/bake.go b/bake/bake.go
index fbea1dc3..86deecfd 100644
--- a/bake/bake.go
+++ b/bake/bake.go
@@ -1117,62 +1117,36 @@ func updateContext(t *build.Inputs, inp *Input) {
 	t.ContextState = &st
 }
 
-// validateContextsEntitlements is a basic check to ensure contexts do not
-// escape local directories when loaded from remote sources. This is to be
-// replaced with proper entitlements support in the future.
-func validateContextsEntitlements(t build.Inputs, inp *Input) error {
-	if inp == nil || inp.State == nil {
-		return nil
-	}
-	if v, ok := os.LookupEnv("BAKE_ALLOW_REMOTE_FS_ACCESS"); ok {
-		if vv, _ := strconv.ParseBool(v); vv {
-			return nil
-		}
-	}
+func collectLocalPaths(t build.Inputs) []string {
+	var out []string
 	if t.ContextState == nil {
-		if err := checkPath(t.ContextPath); err != nil {
-			return err
+		if v, ok := isLocalPath(t.ContextPath); ok {
+			out = append(out, v)
+		}
+		if v, ok := isLocalPath(t.DockerfilePath); ok {
+			out = append(out, v)
+		}
+	} else {
+		if strings.HasPrefix(t.ContextPath, "cwd://") {
+			out = append(out, strings.TrimPrefix(t.ContextPath, "cwd://"))
 		}
 	}
 	for _, v := range t.NamedContexts {
 		if v.State != nil {
 			continue
 		}
-		if err := checkPath(v.Path); err != nil {
-			return err
+		if v, ok := isLocalPath(v.Path); ok {
+			out = append(out, v)
 		}
 	}
-	return nil
+	return out
 }
 
-func checkPath(p string) error {
+func isLocalPath(p string) (string, bool) {
 	if build.IsRemoteURL(p) || strings.HasPrefix(p, "target:") || strings.HasPrefix(p, "docker-image:") {
-		return nil
+		return "", false
 	}
-	p, err := filepath.EvalSymlinks(p)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil
-		}
-		return err
-	}
-	p, err = filepath.Abs(p)
-	if err != nil {
-		return err
-	}
-	wd, err := os.Getwd()
-	if err != nil {
-		return err
-	}
-	rel, err := filepath.Rel(wd, p)
-	if err != nil {
-		return err
-	}
-	parts := strings.Split(rel, string(os.PathSeparator))
-	if parts[0] == ".." {
-		return errors.Errorf("path %s is outside of the working directory, please set BAKE_ALLOW_REMOTE_FS_ACCESS=1", p)
-	}
-	return nil
+	return strings.TrimPrefix(p, "cwd://"), true
 }
 
 func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
@@ -1212,9 +1186,6 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
 		// it's not outside the working directory and then resolve it to an
 		// absolute path.
 		bi.DockerfilePath = path.Clean(strings.TrimPrefix(bi.DockerfilePath, "cwd://"))
-		if err := checkPath(bi.DockerfilePath); err != nil {
-			return nil, err
-		}
 		var err error
 		bi.DockerfilePath, err = filepath.Abs(bi.DockerfilePath)
 		if err != nil {
@@ -1251,10 +1222,6 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
 		}
 	}
 
-	if err := validateContextsEntitlements(bi, inp); err != nil {
-		return nil, err
-	}
-
 	t.Context = &bi.ContextPath
 
 	args := map[string]string{}
diff --git a/bake/entitlements.go b/bake/entitlements.go
index f1eddcfb..ee154be9 100644
--- a/bake/entitlements.go
+++ b/bake/entitlements.go
@@ -108,6 +108,10 @@ func (c EntitlementConf) check(bo build.Options, expected *EntitlementConf) erro
 	rwPaths := map[string]struct{}{}
 	roPaths := map[string]struct{}{}
 
+	for _, p := range collectLocalPaths(bo.Inputs) {
+		roPaths[p] = struct{}{}
+	}
+
 	for _, out := range bo.Exports {
 		if out.Type == "local" {
 			if dest, ok := out.Attrs["dest"]; ok {
@@ -167,7 +171,7 @@ func (c EntitlementConf) check(bo build.Options, expected *EntitlementConf) erro
 	return nil
 }
 
-func (c EntitlementConf) Prompt(ctx context.Context, out io.Writer) error {
+func (c EntitlementConf) Prompt(ctx context.Context, isRemote bool, out io.Writer) error {
 	var term bool
 	if _, err := console.ConsoleFromFile(os.Stdin); err == nil {
 		term = true
@@ -195,6 +199,17 @@ func (c EntitlementConf) Prompt(ctx context.Context, out io.Writer) error {
 	}
 
 	roPaths, rwPaths, commonPaths := groupSamePaths(c.FSRead, c.FSWrite)
+	wd, err := os.Getwd()
+	if err != nil {
+		return errors.Wrap(err, "failed to get current working directory")
+	}
+	wd, err = filepath.EvalSymlinks(wd)
+	if err != nil {
+		return errors.Wrap(err, "failed to evaluate working directory")
+	}
+	roPaths = toRelativePaths(roPaths, wd)
+	rwPaths = toRelativePaths(rwPaths, wd)
+	commonPaths = toRelativePaths(commonPaths, wd)
 
 	if len(commonPaths) > 0 {
 		for _, p := range commonPaths {
@@ -251,6 +266,17 @@ func (c EntitlementConf) Prompt(ctx context.Context, out io.Writer) error {
 	}
 
 	fsEntitlementsEnabled := false
+	if isRemote {
+		if v, ok := os.LookupEnv("BAKE_ALLOW_REMOTE_FS_ACCESS"); ok {
+			vv, err := strconv.ParseBool(v)
+			if err != nil {
+				return errors.Wrapf(err, "failed to parse BAKE_ALLOW_REMOTE_FS_ACCESS value %q", v)
+			}
+			fsEntitlementsEnabled = !vv
+		} else {
+			fsEntitlementsEnabled = true
+		}
+	}
 	v, fsEntitlementsSet := os.LookupEnv("BUILDX_BAKE_ENTITLEMENTS_FS")
 	if fsEntitlementsSet {
 		vv, err := strconv.ParseBool(v)
@@ -334,11 +360,6 @@ loop0:
 }
 
 func dedupPaths(in map[string]struct{}) (map[string]struct{}, error) {
-	wd, err := os.Getwd()
-	if err != nil {
-		return nil, err
-	}
-
 	arr := make([]string, 0, len(in))
 	for p := range in {
 		arr = append(arr, filepath.Clean(p))
@@ -358,18 +379,23 @@ loop0:
 		}
 		m[p] = struct{}{}
 	}
+	return m, nil
+}
 
-	for p := range m {
+func toRelativePaths(in []string, wd string) []string {
+	out := make([]string, 0, len(in))
+	for _, p := range in {
 		rel, err := filepath.Rel(wd, p)
 		if err == nil {
-			if !strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
-				delete(m, p)
-				m[rel] = struct{}{}
+			// allow up to one level of ".." in the path
+			if !strings.HasPrefix(rel, ".."+string(filepath.Separator)+"..") {
+				out = append(out, rel)
+				continue
 			}
 		}
+		out = append(out, p)
 	}
-
-	return m, nil
+	return out
 }
 
 func groupSamePaths(in1, in2 []string) ([]string, []string, []string) {
diff --git a/bake/entitlements_test.go b/bake/entitlements_test.go
index 95f6f434..191d2ef4 100644
--- a/bake/entitlements_test.go
+++ b/bake/entitlements_test.go
@@ -10,6 +10,7 @@ import (
 	"github.com/docker/buildx/build"
 	"github.com/docker/buildx/controller/pb"
 	"github.com/moby/buildkit/client"
+	"github.com/moby/buildkit/client/llb"
 	"github.com/moby/buildkit/util/entitlements"
 	"github.com/stretchr/testify/require"
 )
@@ -132,15 +133,17 @@ func TestDedupePaths(t *testing.T) {
 		},
 		{
 			in: map[string]struct{}{
-				filepath.Join(wd, "a/b/c"):   {},
-				filepath.Join(wd, "../aa"):   {},
-				filepath.Join(wd, "a/b"):     {},
-				filepath.Join(wd, "a/b/d"):   {},
-				filepath.Join(wd, "../aa/b"): {},
+				filepath.Join(wd, "a/b/c"):    {},
+				filepath.Join(wd, "../aa"):    {},
+				filepath.Join(wd, "a/b"):      {},
+				filepath.Join(wd, "a/b/d"):    {},
+				filepath.Join(wd, "../aa/b"):  {},
+				filepath.Join(wd, "../../bb"): {},
 			},
 			out: map[string]struct{}{
-				"a/b":                      {},
-				filepath.Join(wd, "../aa"): {},
+				"a/b":                         {},
+				"../aa":                       {},
+				filepath.Join(wd, "../../bb"): {},
 			},
 		},
 	}
@@ -151,7 +154,18 @@ func TestDedupePaths(t *testing.T) {
 			if err != nil {
 				require.NoError(t, err)
 			}
-			require.Equal(t, tc.out, out)
+			// convert to relative paths as that is shown to user
+			arr := make([]string, 0, len(out))
+			for k := range out {
+				arr = append(arr, k)
+			}
+			require.NoError(t, err)
+			arr = toRelativePaths(arr, wd)
+			m := make(map[string]struct{})
+			for _, v := range arr {
+				m[v] = struct{}{}
+			}
+			require.Equal(t, tc.out, m)
 		})
 	}
 }
@@ -164,6 +178,9 @@ func TestValidateEntitlements(t *testing.T) {
 	err := os.Symlink("../../aa", escapeLink)
 	require.NoError(t, err)
 
+	wd, err := os.Getwd()
+	require.NoError(t, err)
+
 	tcases := []struct {
 		name     string
 		conf     EntitlementConf
@@ -172,6 +189,11 @@ func TestValidateEntitlements(t *testing.T) {
 	}{
 		{
 			name: "No entitlements",
+			opt: build.Options{
+				Inputs: build.Inputs{
+					ContextState: &llb.State{},
+				},
+			},
 		},
 		{
 			name: "NetworkHostMissing",
@@ -182,6 +204,7 @@ func TestValidateEntitlements(t *testing.T) {
 			},
 			expected: EntitlementConf{
 				NetworkHost: true,
+				FSRead:      []string{wd},
 			},
 		},
 		{
@@ -194,7 +217,9 @@ func TestValidateEntitlements(t *testing.T) {
 					entitlements.EntitlementNetworkHost,
 				},
 			},
-			expected: EntitlementConf{},
+			expected: EntitlementConf{
+				FSRead: []string{wd},
+			},
 		},
 		{
 			name: "SecurityAndNetworkHostMissing",
@@ -207,6 +232,7 @@ func TestValidateEntitlements(t *testing.T) {
 			expected: EntitlementConf{
 				NetworkHost:      true,
 				SecurityInsecure: true,
+				FSRead:           []string{wd},
 			},
 		},
 		{
@@ -222,6 +248,7 @@ func TestValidateEntitlements(t *testing.T) {
 			},
 			expected: EntitlementConf{
 				SecurityInsecure: true,
+				FSRead:           []string{wd},
 			},
 		},
 		{
@@ -234,7 +261,8 @@ func TestValidateEntitlements(t *testing.T) {
 				},
 			},
 			expected: EntitlementConf{
-				SSH: true,
+				SSH:    true,
+				FSRead: []string{wd},
 			},
 		},
 		{
@@ -267,6 +295,7 @@ func TestValidateEntitlements(t *testing.T) {
 					slices.Sort(exp)
 					return exp
 				}(),
+				FSRead: []string{wd},
 			},
 		},
 		{
@@ -279,7 +308,7 @@ func TestValidateEntitlements(t *testing.T) {
 				},
 			},
 			conf: EntitlementConf{
-				FSRead: []string{dir1},
+				FSRead: []string{wd, dir1},
 			},
 		},
 		{
@@ -292,7 +321,7 @@ func TestValidateEntitlements(t *testing.T) {
 				},
 			},
 			conf: EntitlementConf{
-				FSRead: []string{dir1},
+				FSRead: []string{wd, dir1},
 			},
 			expected: EntitlementConf{
 				FSRead: []string{filepath.Join(dir1, "../..")},
diff --git a/commands/bake.go b/commands/bake.go
index 4f2bc6f3..ad94c23f 100644
--- a/commands/bake.go
+++ b/commands/bake.go
@@ -257,7 +257,7 @@ func runBake(ctx context.Context, dockerCli command.Cli, targets []string, in ba
 	if err != nil {
 		return err
 	}
-	if err := exp.Prompt(ctx, &syncWriter{w: dockerCli.Err(), wait: printer.Wait}); err != nil {
+	if err := exp.Prompt(ctx, url != "", &syncWriter{w: dockerCli.Err(), wait: printer.Wait}); err != nil {
 		return err
 	}
 	if printer.IsDone() {
diff --git a/tests/bake.go b/tests/bake.go
index a6bad6eb..8f92cb8e 100644
--- a/tests/bake.go
+++ b/tests/bake.go
@@ -508,7 +508,8 @@ EOT
 		withArgs(addr, "--set", "*.output=type=local,dest="+dirDest),
 	)
 	require.Error(t, err, out)
-	require.Contains(t, out, "outside of the working directory, please set BAKE_ALLOW_REMOTE_FS_ACCESS")
+	require.Contains(t, out, "Your build is requesting privileges for following possibly insecure capabilities")
+	require.Contains(t, out, "Read access to path ../")
 
 	out, err = bakeCmd(
 		sb,
@@ -555,7 +556,8 @@ EOT
 		withArgs(addr, "--set", "*.output=type=local,dest="+dirDest),
 	)
 	require.Error(t, err, out)
-	require.Contains(t, out, "outside of the working directory, please set BAKE_ALLOW_REMOTE_FS_ACCESS")
+	require.Contains(t, out, "Your build is requesting privileges for following possibly insecure capabilities")
+	require.Contains(t, out, "Read access to path ..")
 
 	out, err = bakeCmd(
 		sb,