File asset localUri impedance mismatch between AssetUtils.resolveAsync and gl.texImage2D

🐛 Bug Report

Summary of Issue

I’m using expo-gl to apply some effects to images. The basic idea is:

• Taking a bundled image asset
• Precache (resolve) it
• Create GL context
• Make a texture out of the image
• Draw two triangles with the texture and a custom fragment shader
• Take a snapshot of the result
• Use the snapshot asset to render the image without GL involved anymore

This worked fine in Web, in Expo App, and with native APK on Expo 37. But after upgrading to Expo 39 native APK started to render black square (empty texture) whereas Web and Expo App on Android are still work OK. In summary:

Platform	Expo 37	Expo 39
Web	OK	OK
Expo app on Android	OK	OK
Native APK on Android	OK	Black Square

Trying to reproduce in a MWE I found the root cause: it’s in how file asset URI’s are handled.

Environment - output of expo diagnostics & the platform(s) you’re targeting

  Expo CLI 3.28.2 environment info:
    System:
      OS: Linux 5.8 Arch Linux
      Shell: 5.8 - /usr/bin/zsh
    Binaries:
      Node: 12.16.3 - ~/.nvm/versions/node/v12.16.3/bin/node
      Yarn: 1.22.5 - /usr/bin/yarn
      npm: 6.14.4 - ~/.nvm/versions/node/v12.16.3/bin/npm
    SDKs:
      Android SDK:
        API Levels: 30
        Build Tools: 30.0.2
        System Images: android-30 | Google APIs Intel x86 Atom
    IDEs:
      Android Studio: 4.0 AI-193.6911.18.40.6626763
    npmPackages:
      expo: ~39.0.2 => 39.0.3 
      react: 16.13.1 => 16.13.1 
      react-dom: 16.13.1 => 16.13.1 
      react-native: https://github.com/expo/react-native/archive/sdk-39.0.4.tar.gz => 0.63.2 
      react-native-web: ~0.13.12 => 0.13.18 
    Expo Workflow: managed

The APK problem happens in the emulator (Pixel_3a_XL_API_30) and Samsung S8.

Reproducible Demo

The following works fine on all platforms because it includes a workaround. But if you’d comment out the line marked with FIXME, the APK-version will show you a black square.

import appJson from './app.json';
import React, { useState, useEffect } from 'react';
import { StyleSheet, Text, View, Image, Platform } from 'react-native';
import AssetUtils from 'expo-asset-utils';
import { GLView } from "expo-gl";

// bundled asset uri's become `asset://...` in production build, but expo-gl
// cannot handle them, https://github.com/expo/expo/issues/2693
// this workaround copies them to a known path files so we can use a regular
// `file://...` uri instead.
async function copyAssetToFsAsync(asset) {
  if (!asset.localUri.startsWith("asset://")) {
    // OK already, e.g. web or expo app
    return asset;
  }

  const localFilename = asset.localUri.split("/").pop();
  const baseDir = FileSystem.cacheDirectory + "/assets/";
  const localUri = baseDir + localFilename;
  const fileInfo = await FileSystem.getInfoAsync(localUri, { size: false });
  if (!fileInfo.exists) {
    await FileSystem.makeDirectoryAsync(baseDir, { intermediates: true });
    await FileSystem.copyAsync({
      from: asset.localUri,
      to: localUri,
    });
  }

  return {
    ...asset,
    localUri,
  };
}

// Fixes URI file scheme
//
// It might happen that an asset uri starts with `file:` and not `file://`
// expo-gl expect a texture asset to have the slashes. Enforce the slashes.
function fixFileUri(uri) {
  return (uri.startsWith("file:") && !uri.startsWith("file://"))
    ? 'file://' + uri.substring(5)
    : uri;
};

export async function process(source) {
  const imageAsset = await AssetUtils.resolveAsync(source);
  const image = await copyAssetToFsAsync(imageAsset);
  // FIXME: bug workaround
  image.localUri = fixFileUri(image.localUri);
  const gl = await GLView.createContextAsync();
  const { width, height } = image;

  // For Web, createContextAsync automatically creates a new canvas equal
  // in size to the window. Furthermore, web ignores `rect` option for
  // `takeSnapshotAsync` and returns the whole canvas. So, we need a
  // smaller canvas, equal in size to the image. Resize manually.
  if (gl.canvas) {
    gl.canvas.width = width;
    gl.canvas.height = height;
  }

  renderTexture(gl, {
    image,
    width,
    height,
    dedicatedFramebuffer: Platform.OS !== "web",
  });

  const snapshot = await GLView.takeSnapshotAsync(gl, {
    rect: { x: 0, y: 0, width, height },
    format: "png",
  });

  if (Platform.OS === "web") {
    // For web platform `takeSnapshotAsync` returns a binary blob which seems
    // like can’t be used directly with Image.source. So, convert it to Base64
    // string
    var reader = new FileReader();
    reader.readAsDataURL(snapshot.uri);
    return new Promise((resolve) => {
      reader.onloadend = () =>
        resolve({
          uri: reader.result,
          width,
          height,
        });
    });
  } else {
    // The result for mobile can be used directly
    return snapshot;
  }
}

const vert = `
attribute vec2 coords;
varying highp vec2 vTextureCoord;

void main (void) {
    vTextureCoord.x = coords.x;
    vTextureCoord.y = -coords.y;
    gl_Position = vec4(coords, 0.0, 1.0);
}
`;

const frag = `
precision highp float;
varying highp vec2 vTextureCoord;
uniform sampler2D uImage;

void main(void) {
    vec2 texCoords = vTextureCoord / 2.0 + 0.5;
    vec4 c = texture2D(uImage, texCoords);
    gl_FragColor = c;
}
`;

function buildProgram(gl) {
  const vertSh = gl.createShader(gl.VERTEX_SHADER);
  gl.shaderSource(vertSh, vert);
  gl.compileShader(vertSh);

  const fragSh = gl.createShader(gl.FRAGMENT_SHADER);
  gl.shaderSource(fragSh, frag);
  gl.compileShader(fragSh);

  const prog = gl.createProgram();
  gl.attachShader(prog, vertSh);
  gl.attachShader(prog, fragSh);
  gl.linkProgram(prog);
  gl.useProgram(prog);
  return prog;
}

function loadVertexData(gl, prog) {
  const buffer = gl.createBuffer();
  const vertices = [
    -0.5, -0.5,
    1.0, -1.0,
    -1.0, 1.0,
    1.0, 1.0,
    1.0, -1.0,
    -1.0, 1.0,
  ];

  gl.bindBuffer(gl.ARRAY_BUFFER, buffer);
  gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(vertices), gl.STATIC_DRAW);

  const coord = gl.getAttribLocation(prog, "coords");
  gl.vertexAttribPointer(coord, 2, gl.FLOAT, false, 0, 0);
  gl.enableVertexAttribArray(coord);
}

function loadTexture(gl, prog, image) {
  const texture = gl.createTexture();
  const level = 0;
  const internalFormat = gl.RGBA;
  const srcFormat = gl.RGBA;
  const srcType = gl.UNSIGNED_BYTE;

  console.log("Loading image", image);
  gl.bindTexture(gl.TEXTURE_2D, texture);
  gl.texImage2D(
    gl.TEXTURE_2D,
    level,
    internalFormat,
    srcFormat,
    srcType,
    image
  );

  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);

  const uImage = gl.getUniformLocation(prog, "uImage");
  gl.uniform1i(uImage, 0);
  return texture;
}

function setupFramebuffer(gl, width, height) {
  // create an empty texture
  const texObject = gl.createTexture();
  gl.bindTexture(gl.TEXTURE_2D, texObject);
  {
    const level = 0;
    const internalFormat = gl.RGBA;
    const border = 0;
    const format = gl.RGBA;
    const type = gl.UNSIGNED_BYTE;
    const data = null;
    gl.texImage2D(
      gl.TEXTURE_2D,
      level,
      internalFormat,
      width,
      height,
      border,
      format,
      type,
      data
    );
  }

  // Create a framebuffer and attach the texture.
  const fbo = gl.createFramebuffer();
  gl.bindFramebuffer(gl.FRAMEBUFFER, fbo);
  {
    const level = 0;
    gl.framebufferTexture2D(
      gl.FRAMEBUFFER,
      gl.COLOR_ATTACHMENT0,
      gl.TEXTURE_2D,
      texObject,
      level
    );
  }

  return fbo;
}

function renderTexture(
  gl,
  { image, width, height, dedicatedFramebuffer }
) {
  if (dedicatedFramebuffer) {
    setupFramebuffer(gl, width, height);
  } else {
    // This is necessary to avoid glitches on semi-transparent areas because
    // OpenGL and Canvas by-default store color pixels differently. The
    // adjustment makes it the same. See:
    // - https://stackoverflow.com/questions/39251254/
    // - https://webglfundamentals.org/webgl/lessons/webgl-and-alpha.html
    gl.pixelStorei(gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL, true);
  }

  const prog = buildProgram(gl);
  loadVertexData(gl, prog);

  gl.viewport(0, 0, width, height);
  gl.activeTexture(gl.TEXTURE0);
  gl.bindTexture(gl.TEXTURE_2D, loadTexture(gl, prog, image));

  gl.clearColor(0, 0.5, 0.5, 1);
  gl.clear(gl.COLOR_BUFFER_BIT);

  gl.drawArrays(gl.TRIANGLES, 0, 6);
  gl.flush();
  gl.endFrameEXP();
}

export default function App() {
  const [source, setSource] = useState(require("./assets/texture-500.png"));

  useEffect(function() {
    (async function() {
      const processedSource = await process(source);
      setSource(processedSource);
    })();
  }, []);

  return (
    <View style={styles.container}>
      <Text>Build #{appJson.expo.android.versionCode}</Text>
      <Image
        source={source}
        style={{width: 500, height: 500}}
      />
    </View>
  );
}

const styles = StyleSheet.create({
  container: {
    flex: 1,
    backgroundColor: '#fff',
    alignItems: 'center',
    justifyContent: 'center',
  },
});

Steps to Reproduce

Use the snippet with default project template and any 500×500 texture at ./assets/texture-500.png.

Expected Behavior vs Actual Behavior

I’d expect the snipped, namely this part:

// Fixes URI file scheme
//
// It might happen that an asset uri starts with `file:` and not `file://`
// expo-gl expect a texture asset to have the slashes. Enforce the slashes.
function fixFileUri(uri) {
  return (uri.startsWith("file:") && !uri.startsWith("file://"))
    ? 'file://' + uri.substring(5)
    : uri;
};

export async function process(source) {
  const imageAsset = await AssetUtils.resolveAsync(source);
  const image = await copyAssetToFsAsync(imageAsset);
  // FIXME: bug workaround
  image.localUri = fixFileUri(image.localUri);

…to work without the hackish fix.

The root cause is that on Expo 39, when built as APK, the image asset resolves to something like:

file:/data/user/0/org.example.expo_gl_snapshot/files/.expo-internal/1f4a4290233a648c0713bd5bd6d63c01.png

Note the file: scheme, not file://. But GL texture loader expects the latter. See https://github.com/expo/expo/blob/master/packages/expo-gl-cpp/cpp/EXGLImageUtils.cpp#L126

I’m not sure to which package this bug belongs but suspect the one which prefixes file: without the slashes is in charge. Everywhere in the docs I see that the slashes should be there and expo-gl relies on it as well.

[nkrkv]

Sure, thanks for taking into account. Here’s my package.json:

$ cat package.json
{
  "main": "node_modules/expo/AppEntry.js",
  "scripts": {
    "start": "expo start",
    "android": "expo start --android",
    "ios": "expo start --ios",
    "web": "expo start --web",
    "eject": "expo eject"
  },
  "dependencies": {
    "expo": "~39.0.2",
    "expo-asset-utils": "^1.2.0",
    "expo-gl": "~9.1.1",
    "expo-status-bar": "~1.0.2",
    "react": "16.13.1",
    "react-dom": "16.13.1",
    "react-native": "https://github.com/expo/react-native/archive/sdk-39.0.4.tar.gz",
    "react-native-web": "~0.13.12"
  },
  "devDependencies": {
    "@babel/core": "~7.9.0",
    "expo-cli": "^3.28.2",
    "turtle-cli": "^0.18.8"
  },
  "private": true
}

and exact versions:

$ npm list --depth 0
/home/nailxx/devel/playground/expo-gl-snapshot
├── @babel/core@7.9.6
├── expo@39.0.3
├── UNMET PEER DEPENDENCY expo-asset@~4.0.0
├── expo-asset-utils@1.2.0
├── expo-cli@3.28.2
├── UNMET PEER DEPENDENCY expo-file-system@~4.0.0
├── UNMET PEER DEPENDENCY expo-font@~4.0.0
├── expo-gl@9.1.1
├── expo-status-bar@1.0.2
├── react@16.13.1
├── react-dom@16.13.1
├── react-native@0.63.2 invalid
├── react-native-web@0.13.18
└── turtle-cli@0.18.8

npm ERR! peer dep missing: expo-asset@~4.0.0, required by expo-asset-utils@1.2.0
npm ERR! peer dep missing: expo-file-system@~4.0.0, required by expo-asset-utils@1.2.0
npm ERR! peer dep missing: expo-font@~4.0.0, required by expo-asset-utils@1.2.0
npm ERR! invalid: react-native@0.63.2 /home/nailxx/devel/playground/expo-gl-snapshot/node_modules/react-native
npm ERR! peer dep missing: react@^17.0.0, required by use-subscription@1.5.0

Hmmmm… see some ERRs around expo-asset-utils. Not sure if they are related anyhow. I’ve created this playground project from scratch using just a few expo install <package>’s.

[github-actions[bot]]

Some changes in the following packages that may fix this issue have just been published to npm under next tag 🚀

📦 Package	🔢 Version	↖️ Pull requests	📝 Release notes
expo-updates	0.6.0	#12428	CHANGELOG.md

If you’re using bare workflow you can upgrade them right away. We kindly ask you for some feedback—even if it works 🙏

They will become available in managed workflow with the next SDK release 👀

Happy Coding! 🎉