I was able to change the images by first importing them like in the example here
// path: ./my-app/src/admin/app.js
import AuthLogo from './extensions/my-logo.png';
import MenuLogo from './extensions/logo.png';
import favicon from './extensions/favicon.ico';
export default {
config: {
// Replace the Strapi logo in auth (login) views
auth: {
logo: AuthLogo,
},
// Replace the favicon
head: {
favicon: favicon,
},
...