Page updated Jan 16, 2024

Example application

Sample React app

A sample React application with all of the Predictions features is provided below. It shows how to use all scenarios above by calling the appropriate convert, identify, and interpret API calls in the Amplify library.

To use Predictions.convert() with Amazon Transcribe you will need to install the following dependency in your app first:

npm install microphone-stream
1npm install microphone-stream

The components in the app code below are rendered according to the scenarios above like so:

return ( <div className="App"> <TextTranslation /> <TextToSpeech /> <SpeechToText /> <TextIdentification /> <EntityIdentification /> <PredictionsUpload /> <LabelsIdentification /> <TextInterpretation /> </div> );
1return (
2 <div className="App">
3 <TextTranslation />
4 <TextToSpeech />
5 <SpeechToText />
6 <TextIdentification />
7 <EntityIdentification />
8 <PredictionsUpload />
9 <LabelsIdentification />
10 <TextInterpretation />
11 </div>
12);

React app code

import React, { useState } from 'react'; import './App.css'; import { Amplify, Storage, Predictions } from 'aws-amplify'; import { AmazonAIPredictionsProvider } from '@aws-amplify/predictions'; import awsconfig from './aws-exports'; import mic from 'microphone-stream'; Amplify.configure(awsconfig); Amplify.addPluggable(new AmazonAIPredictionsProvider()); function TextIdentification() { const [response, setResponse] = useState("You can add a photo by uploading directly from the app ") function identifyFromFile(event) { setResponse('identifying text...'); const { target: { files } } = event; const [file,] = files || []; if (!file) { return; } Predictions.identify({ text: { source: { file, }, format: "PLAIN", // Available options "PLAIN", "FORM", "TABLE", "ALL" } }).then(({text: { fullText }}) => { setResponse(fullText) }) .catch(err => setResponse(JSON.stringify(err, null, 2))) } return ( <div className="Text"> <div> <h3>Text identification</h3> <input type="file" onChange={identifyFromFile}></input> <p>{response}</p> </div> </div> ); } function EntityIdentification() { const [response, setResponse] = useState("Click upload for test ") const [src, setSrc] = useState(""); function identifyFromFile(event) { setResponse('searching...'); const { target: { files } } = event; const [file,] = files || []; if (!file) { return; } Predictions.identify({ entities: { source: { file, }, /**For using the Identify Entities advanced features, enable collection:true and comment out celebrityDetection * Then after you upload a face with PredictionsUpload you'll be able to run this again * and it will tell you if the photo you're testing is in that Collection or not and display it*/ //collection: true celebrityDetection: true } }).then(result => { console.log(result); const entities = result.entities; let imageId = "" let names = "" entities.forEach(({ boundingBox, metadata: { name = "", externalImageId = "" } }) => { const { width, // ratio of overall image width height, // ratio of overall image height left, // left coordinate as a ratio of overall image width top // top coordinate as a ratio of overall image height } = boundingBox; imageId = externalImageId; if (name) { names += name + " ."; } console.log({ name }); }) if (imageId) { Storage.get("", { customPrefix: { public: imageId }, level: "public", }).then(setSrc); // this should be better but it works } console.log({ entities }); setResponse(names); }) .catch(err => console.log(err)) } return ( <div className="Text"> <div> <h3>Entity identification</h3> <input type="file" onChange={identifyFromFile}></input> <p>{response}</p> { src && <img src="{src}"></img>} </div> </div> ); } function PredictionsUpload() { /* This is Identify Entities Advanced feature * This will upload user images to the appropriate bucket prefix * and a Lambda trigger will automatically perform indexing */ function upload(event) { const { target: { files } } = event; const [file,] = files || []; Storage.put(file.name, file, { level: 'protected', customPrefix: { protected: 'protected/predictions/index-faces/', } }); } return ( <div className="Text"> <div> <h3>Upload to predictions s3</h3> <input type="file" onChange={upload}></input> </div> </div> ); } function LabelsIdentification() { const [response, setResponse] = useState("Click upload for test ") function identifyFromFile(event) { const { target: { files } } = event; const [file,] = files || []; if (!file) { return; } Predictions.identify({ labels: { source: { file, }, type: "ALL" // "LABELS" will detect objects , "UNSAFE" will detect if content is not safe, "ALL" will do both default on aws-exports.js } }).then(result => setResponse(JSON.stringify(result, null, 2))) .catch(err => setResponse(JSON.stringify(err, null, 2))) } return ( <div className="Text"> <div> <h3>Labels identification</h3> <input type="file" onChange={identifyFromFile}></input> <p>{response}</p> </div> </div> ); } function SpeechToText(props) { const [response, setResponse] = useState("Press 'start recording' to begin your transcription. Press STOP recording once you finish speaking.") function AudioRecorder(props) { const [recording, setRecording] = useState(false); const [micStream, setMicStream] = useState(); const [audioBuffer] = useState( (function() { let buffer = []; function add(raw) { buffer = buffer.concat(...raw); return buffer; } function newBuffer() { console.log("resetting buffer"); buffer = []; } return { reset: function() { newBuffer(); }, addData: function(raw) { return add(raw); }, getData: function() { return buffer; } }; })() ); async function startRecording() { console.log('start recording'); audioBuffer.reset(); window.navigator.mediaDevices.getUserMedia({ video: false, audio: true }).then((stream) => { const startMic = new mic(); startMic.setStream(stream); startMic.on('data', (chunk) => { var raw = mic.toRaw(chunk); if (raw == null) { return; } audioBuffer.addData(raw); }); setRecording(true); setMicStream(startMic); }); } async function stopRecording() { console.log('stop recording'); const { finishRecording } = props; micStream.stop(); setMicStream(null); setRecording(false); const resultBuffer = audioBuffer.getData(); if (typeof finishRecording === "function") { finishRecording(resultBuffer); } } return ( <div className="audioRecorder"> <div> {recording && <button onClick={stopRecording}>Stop recording</button>} {!recording && <button onClick={startRecording}>Start recording</button>} </div> </div> ); } function convertFromBuffer(bytes) { setResponse('Converting text...'); Predictions.convert({ transcription: { source: { bytes }, // language: "en-US", // other options are "en-GB", "fr-FR", "fr-CA", "es-US" }, }).then(({ transcription: { fullText } }) => setResponse(fullText)) .catch(err => setResponse(JSON.stringify(err, null, 2))) } return ( <div className="Text"> <div> <h3>Speech to text</h3> <AudioRecorder finishRecording={convertFromBuffer} /> <p>{response}</p> </div> </div> ); } function TextToSpeech() { const [response, setResponse] = useState("...") const [textToGenerateSpeech, setTextToGenerateSpeech] = useState("write to speech"); function generateTextToSpeech() { setResponse('Generating audio...'); Predictions.convert({ textToSpeech: { source: { text: textToGenerateSpeech, }, voiceId: "Amy" // default configured on aws-exports.js // list of different options are here https://docs.aws.amazon.com/polly/latest/dg/voicelist.html } }).then(result => { let AudioContext = window.AudioContext || window.webkitAudioContext; console.log({ AudioContext }); const audioCtx = new AudioContext(); const source = audioCtx.createBufferSource(); audioCtx.decodeAudioData(result.audioStream, (buffer) => { source.buffer = buffer; source.connect(audioCtx.destination); source.start(0); }, (err) => console.log({err})); setResponse(`Generation completed, press play`); }) .catch(err => setResponse(err)) } function setText(event) { setTextToGenerateSpeech(event.target.value); } return ( <div className="Text"> <div> <h3>Text To Speech</h3> <input value={textToGenerateSpeech} onChange={setText}></input> <button onClick={generateTextToSpeech}>Text to Speech</button> <h3>{response}</h3> </div> </div> ); } function TextTranslation() { const [response, setResponse] = useState("Input some text and click enter to test") const [textToTranslate, setTextToTranslate] = useState("write to translate"); function translate() { Predictions.convert({ translateText: { source: { text: textToTranslate, // language : "es" // defaults configured on aws-exports.js // supported languages https://docs.aws.amazon.com/translate/latest/dg/how-it-works.html#how-it-works-language-codes }, // targetLanguage: "en" } }).then(result => setResponse(JSON.stringify(result, null, 2))) .catch(err => setResponse(JSON.stringify(err, null, 2))) } function setText(event) { setTextToTranslate(event.target.value); } return ( <div className="Text"> <div> <h3>Text Translation</h3> <input value={textToTranslate} onChange={setText}></input> <button onClick={translate}>Translate</button> <p>{response}</p> </div> </div> ); } function TextInterpretation() { const [response, setResponse] = useState("Input some text and click enter to test") const [textToInterpret, setTextToInterpret] = useState("write some text here to interpret"); function interpretFromPredictions() { Predictions.interpret({ text: { source: { text: textToInterpret, }, type: "ALL" } }).then(result => setResponse(JSON.stringify(result, null, 2))) .catch(err => setResponse(JSON.stringify(err, null, 2))) } function setText(event) { setTextToInterpret(event.target.value); } return ( <div className="Text"> <div> <h3>Text interpretation</h3> <input value={textToInterpret} onChange={setText}></input> <button onClick={interpretFromPredictions}>test</button> <p>{response}</p> </div> </div> ); } function App() { return ( <div className="App"> Translate Text <TextTranslation /> <br/> Speech Generation <TextToSpeech /> <br/> Transcribe Audio <SpeechToText /> <br/> Identify Text <TextIdentification /> <br/> Identify Entities <EntityIdentification /> <br/> Identify Entities (Advanced) <PredictionsUpload /> <br/> Label Objects <LabelsIdentification /> <br/> Text Interpretation <TextInterpretation /> </div> ); } export default App;
1import React, { useState } from 'react';
2
3import './App.css';
4import { Amplify, Storage, Predictions } from 'aws-amplify';
5import { AmazonAIPredictionsProvider } from '@aws-amplify/predictions';
6
7import awsconfig from './aws-exports';
8
9import mic from 'microphone-stream';
10
11Amplify.configure(awsconfig);
12Amplify.addPluggable(new AmazonAIPredictionsProvider());
13
14function TextIdentification() {
15 const [response, setResponse] = useState("You can add a photo by uploading directly from the app ")
16
17 function identifyFromFile(event) {
18 setResponse('identifying text...');
19 const { target: { files } } = event;
20 const [file,] = files || [];
21
22 if (!file) {
23 return;
24 }
25 Predictions.identify({
26 text: {
27 source: {
28 file,
29 },
30 format: "PLAIN", // Available options "PLAIN", "FORM", "TABLE", "ALL"
31 }
32 }).then(({text: { fullText }}) => {
33 setResponse(fullText)
34 })
35 .catch(err => setResponse(JSON.stringify(err, null, 2)))
36 }
37
38 return (
39 <div className="Text">
40 <div>
41 <h3>Text identification</h3>
42 <input type="file" onChange={identifyFromFile}></input>
43 <p>{response}</p>
44 </div>
45 </div>
46 );
47}
48
49function EntityIdentification() {
50 const [response, setResponse] = useState("Click upload for test ")
51 const [src, setSrc] = useState("");
52
53 function identifyFromFile(event) {
54 setResponse('searching...');
55
56 const { target: { files } } = event;
57 const [file,] = files || [];
58
59 if (!file) {
60 return;
61 }
62 Predictions.identify({
63 entities: {
64 source: {
65 file,
66 },
67 /**For using the Identify Entities advanced features, enable collection:true and comment out celebrityDetection
68 * Then after you upload a face with PredictionsUpload you'll be able to run this again
69 * and it will tell you if the photo you're testing is in that Collection or not and display it*/
70 //collection: true
71 celebrityDetection: true
72 }
73 }).then(result => {
74 console.log(result);
75 const entities = result.entities;
76 let imageId = ""
77 let names = ""
78 entities.forEach(({ boundingBox, metadata: { name = "", externalImageId = "" } }) => {
79 const {
80 width, // ratio of overall image width
81 height, // ratio of overall image height
82 left, // left coordinate as a ratio of overall image width
83 top // top coordinate as a ratio of overall image height
84 } = boundingBox;
85 imageId = externalImageId;
86 if (name) {
87 names += name + " .";
88 }
89 console.log({ name });
90 })
91 if (imageId) {
92 Storage.get("", {
93 customPrefix: {
94 public: imageId
95 },
96 level: "public",
97 }).then(setSrc); // this should be better but it works
98 }
99 console.log({ entities });
100 setResponse(names);
101 })
102 .catch(err => console.log(err))
103 }
104
105 return (
106 <div className="Text">
107 <div>
108 <h3>Entity identification</h3>
109 <input type="file" onChange={identifyFromFile}></input>
110 <p>{response}</p>
111 { src && <img src="{src}"></img>}
112 </div>
113 </div>
114 );
115}
116
117function PredictionsUpload() {
118 /* This is Identify Entities Advanced feature
119 * This will upload user images to the appropriate bucket prefix
120 * and a Lambda trigger will automatically perform indexing
121 */
122 function upload(event) {
123 const { target: { files } } = event;
124 const [file,] = files || [];
125 Storage.put(file.name, file, {
126 level: 'protected',
127 customPrefix: {
128 protected: 'protected/predictions/index-faces/',
129 }
130 });
131 }
132
133 return (
134 <div className="Text">
135 <div>
136 <h3>Upload to predictions s3</h3>
137 <input type="file" onChange={upload}></input>
138 </div>
139 </div>
140 );
141}
142
143function LabelsIdentification() {
144 const [response, setResponse] = useState("Click upload for test ")
145
146 function identifyFromFile(event) {
147 const { target: { files } } = event;
148 const [file,] = files || [];
149
150 if (!file) {
151 return;
152 }
153 Predictions.identify({
154 labels: {
155 source: {
156 file,
157 },
158 type: "ALL" // "LABELS" will detect objects , "UNSAFE" will detect if content is not safe, "ALL" will do both default on aws-exports.js
159 }
160 }).then(result => setResponse(JSON.stringify(result, null, 2)))
161 .catch(err => setResponse(JSON.stringify(err, null, 2)))
162 }
163
164 return (
165 <div className="Text">
166 <div>
167 <h3>Labels identification</h3>
168 <input type="file" onChange={identifyFromFile}></input>
169 <p>{response}</p>
170 </div>
171 </div>
172 );
173}
174
175function SpeechToText(props) {
176 const [response, setResponse] = useState("Press 'start recording' to begin your transcription. Press STOP recording once you finish speaking.")
177
178 function AudioRecorder(props) {
179 const [recording, setRecording] = useState(false);
180 const [micStream, setMicStream] = useState();
181 const [audioBuffer] = useState(
182 (function() {
183 let buffer = [];
184 function add(raw) {
185 buffer = buffer.concat(...raw);
186 return buffer;
187 }
188 function newBuffer() {
189 console.log("resetting buffer");
190 buffer = [];
191 }
192
193 return {
194 reset: function() {
195 newBuffer();
196 },
197 addData: function(raw) {
198 return add(raw);
199 },
200 getData: function() {
201 return buffer;
202 }
203 };
204 })()
205 );
206
207 async function startRecording() {
208 console.log('start recording');
209 audioBuffer.reset();
210
211 window.navigator.mediaDevices.getUserMedia({ video: false, audio: true }).then((stream) => {
212 const startMic = new mic();
213
214 startMic.setStream(stream);
215 startMic.on('data', (chunk) => {
216 var raw = mic.toRaw(chunk);
217 if (raw == null) {
218 return;
219 }
220 audioBuffer.addData(raw);
221
222 });
223
224 setRecording(true);
225 setMicStream(startMic);
226 });
227 }
228
229 async function stopRecording() {
230 console.log('stop recording');
231 const { finishRecording } = props;
232
233 micStream.stop();
234 setMicStream(null);
235 setRecording(false);
236
237 const resultBuffer = audioBuffer.getData();
238
239 if (typeof finishRecording === "function") {
240 finishRecording(resultBuffer);
241 }
242
243 }
244
245 return (
246 <div className="audioRecorder">
247 <div>
248 {recording && <button onClick={stopRecording}>Stop recording</button>}
249 {!recording && <button onClick={startRecording}>Start recording</button>}
250 </div>
251 </div>
252 );
253 }
254
255 function convertFromBuffer(bytes) {
256 setResponse('Converting text...');
257
258 Predictions.convert({
259 transcription: {
260 source: {
261 bytes
262 },
263 // language: "en-US", // other options are "en-GB", "fr-FR", "fr-CA", "es-US"
264 },
265 }).then(({ transcription: { fullText } }) => setResponse(fullText))
266 .catch(err => setResponse(JSON.stringify(err, null, 2)))
267 }
268
269 return (
270 <div className="Text">
271 <div>
272 <h3>Speech to text</h3>
273 <AudioRecorder finishRecording={convertFromBuffer} />
274 <p>{response}</p>
275 </div>
276 </div>
277 );
278}
279
280function TextToSpeech() {
281 const [response, setResponse] = useState("...")
282 const [textToGenerateSpeech, setTextToGenerateSpeech] = useState("write to speech");
283
284 function generateTextToSpeech() {
285 setResponse('Generating audio...');
286 Predictions.convert({
287 textToSpeech: {
288 source: {
289 text: textToGenerateSpeech,
290 },
291 voiceId: "Amy" // default configured on aws-exports.js
292 // list of different options are here https://docs.aws.amazon.com/polly/latest/dg/voicelist.html
293 }
294 }).then(result => {
295 let AudioContext = window.AudioContext || window.webkitAudioContext;
296 console.log({ AudioContext });
297 const audioCtx = new AudioContext();
298 const source = audioCtx.createBufferSource();
299 audioCtx.decodeAudioData(result.audioStream, (buffer) => {
300
301 source.buffer = buffer;
302 source.connect(audioCtx.destination);
303 source.start(0);
304 }, (err) => console.log({err}));
305
306 setResponse(`Generation completed, press play`);
307 })
308 .catch(err => setResponse(err))
309 }
310
311 function setText(event) {
312 setTextToGenerateSpeech(event.target.value);
313 }
314
315 return (
316 <div className="Text">
317 <div>
318 <h3>Text To Speech</h3>
319 <input value={textToGenerateSpeech} onChange={setText}></input>
320 <button onClick={generateTextToSpeech}>Text to Speech</button>
321 <h3>{response}</h3>
322 </div>
323 </div>
324 );
325}
326
327function TextTranslation() {
328 const [response, setResponse] = useState("Input some text and click enter to test")
329 const [textToTranslate, setTextToTranslate] = useState("write to translate");
330
331 function translate() {
332 Predictions.convert({
333 translateText: {
334 source: {
335 text: textToTranslate,
336 // language : "es" // defaults configured on aws-exports.js
337 // supported languages https://docs.aws.amazon.com/translate/latest/dg/how-it-works.html#how-it-works-language-codes
338 },
339 // targetLanguage: "en"
340 }
341 }).then(result => setResponse(JSON.stringify(result, null, 2)))
342 .catch(err => setResponse(JSON.stringify(err, null, 2)))
343 }
344
345 function setText(event) {
346 setTextToTranslate(event.target.value);
347 }
348
349 return (
350 <div className="Text">
351 <div>
352 <h3>Text Translation</h3>
353 <input value={textToTranslate} onChange={setText}></input>
354 <button onClick={translate}>Translate</button>
355 <p>{response}</p>
356 </div>
357 </div>
358 );
359}
360
361function TextInterpretation() {
362 const [response, setResponse] = useState("Input some text and click enter to test")
363 const [textToInterpret, setTextToInterpret] = useState("write some text here to interpret");
364
365 function interpretFromPredictions() {
366 Predictions.interpret({
367 text: {
368 source: {
369 text: textToInterpret,
370 },
371 type: "ALL"
372 }
373 }).then(result => setResponse(JSON.stringify(result, null, 2)))
374 .catch(err => setResponse(JSON.stringify(err, null, 2)))
375 }
376
377 function setText(event) {
378 setTextToInterpret(event.target.value);
379 }
380
381 return (
382 <div className="Text">
383 <div>
384 <h3>Text interpretation</h3>
385 <input value={textToInterpret} onChange={setText}></input>
386 <button onClick={interpretFromPredictions}>test</button>
387 <p>{response}</p>
388 </div>
389 </div>
390 );
391}
392
393function App() {
394 return (
395 <div className="App">
396 Translate Text
397 <TextTranslation />
398 <br/>
399 Speech Generation
400 <TextToSpeech />
401 <br/>
402 Transcribe Audio
403 <SpeechToText />
404 <br/>
405 Identify Text
406 <TextIdentification />
407 <br/>
408 Identify Entities
409 <EntityIdentification />
410 <br/>
411 Identify Entities (Advanced)
412 <PredictionsUpload />
413 <br/>
414 Label Objects
415 <LabelsIdentification />
416 <br/>
417 Text Interpretation
418 <TextInterpretation />
419 </div>
420 );
421}
422
423export default App;

Now run npm start and press the buttons to demo the app.