Example application

1npm install microphone-stream

1return (
2  <div className="App">
3    <TextTranslation />
4    <TextToSpeech />
5    <SpeechToText />
6    <TextIdentification />
7    <EntityIdentification />
8    <PredictionsUpload />
9    <LabelsIdentification />
10    <TextInterpretation />
11  </div>
12);

1import React, { useState } from 'react';
2
3import './App.css';
4import { Amplify, Storage, Predictions } from 'aws-amplify';
5import { AmazonAIPredictionsProvider } from '@aws-amplify/predictions';
6
7import awsconfig from './aws-exports';
8
9import mic from 'microphone-stream';
10
11Amplify.configure(awsconfig);
12Amplify.addPluggable(new AmazonAIPredictionsProvider());
13
14function TextIdentification() {
15  const [response, setResponse] = useState("You can add a photo by uploading directly from the app ")
16
17  function identifyFromFile(event) {
18    setResponse('identifying text...');
19    const { target: { files } } = event;
20    const [file,] = files || [];
21
22    if (!file) {
23      return;
24    }
25    Predictions.identify({
26      text: {
27        source: {
28          file,
29        },
30        format: "PLAIN", // Available options "PLAIN", "FORM", "TABLE", "ALL"
31      }
32    }).then(({text: { fullText }}) => {
33      setResponse(fullText)
34    })
35      .catch(err => setResponse(JSON.stringify(err, null, 2)))
36  }
37
38  return (
39    <div className="Text">
40      <div>
41        <h3>Text identification</h3>
42        <input type="file" onChange={identifyFromFile}></input>
43        <p>{response}</p>
44      </div>
45    </div>
46  );
47}
48
49function EntityIdentification() {
50  const [response, setResponse] = useState("Click upload for test ")
51  const [src, setSrc] = useState("");
52
53  function identifyFromFile(event) {
54    setResponse('searching...');
55    
56    const { target: { files } } = event;
57    const [file,] = files || [];
58
59    if (!file) {
60      return;
61    }
62    Predictions.identify({
63      entities: {
64        source: {
65          file,
66        },
67        /**For using the Identify Entities advanced features, enable collection:true and comment out celebrityDetection
68         * Then after you upload a face with PredictionsUpload you'll be able to run this again
69         * and it will tell you if the photo you're testing is in that Collection or not and display it*/
70        //collection: true
71        celebrityDetection: true
72      }
73    }).then(result => {
74      console.log(result);
75      const entities = result.entities;
76      let imageId = ""
77      let names = ""
78      entities.forEach(({ boundingBox, metadata: { name = "", externalImageId = "" } }) => {
79        const {
80          width, // ratio of overall image width
81          height, // ratio of overall image height
82          left, // left coordinate as a ratio of overall image width
83          top // top coordinate as a ratio of overall image height
84        } = boundingBox;
85        imageId = externalImageId;
86        if (name) {
87          names += name + " .";
88        }
89        console.log({ name });
90      })
91      if (imageId) {
92        Storage.get("", {
93          customPrefix: {
94            public: imageId
95          },
96          level: "public",
97        }).then(setSrc); // this should be better but it works
98      }
99      console.log({ entities });
100      setResponse(names);
101    })
102      .catch(err => console.log(err))
103  }
104
105  return (
106    <div className="Text">
107      <div>
108        <h3>Entity identification</h3>
109        <input type="file" onChange={identifyFromFile}></input>
110        <p>{response}</p>
111        { src && <img src="{src}"></img>}
112      </div>
113    </div>
114  );
115}
116
117function PredictionsUpload() {
118  /* This is Identify Entities Advanced feature
119   * This will upload user images to the appropriate bucket prefix
120   * and a Lambda trigger will automatically perform indexing
121   */
122  function upload(event) {
123    const { target: { files } } = event;
124    const [file,] = files || [];
125    Storage.put(file.name, file, {
126      level: 'protected',
127      customPrefix: {
128        protected: 'protected/predictions/index-faces/',
129      }
130    });
131  }
132
133  return (
134    <div className="Text">
135      <div>
136        <h3>Upload to predictions s3</h3>
137        <input type="file" onChange={upload}></input>
138      </div>
139    </div>
140  );
141}
142
143function LabelsIdentification() {
144  const [response, setResponse] = useState("Click upload for test ")
145
146  function identifyFromFile(event) {
147    const { target: { files } } = event;
148    const [file,] = files || [];
149
150    if (!file) {
151      return;
152    }
153    Predictions.identify({
154      labels: {
155        source: {
156          file,
157        },
158        type: "ALL" // "LABELS" will detect objects , "UNSAFE" will detect if content is not safe, "ALL" will do both default on aws-exports.js
159      }
160    }).then(result => setResponse(JSON.stringify(result, null, 2)))
161      .catch(err => setResponse(JSON.stringify(err, null, 2)))
162  }
163
164  return (
165    <div className="Text">
166      <div>
167        <h3>Labels identification</h3>
168        <input type="file" onChange={identifyFromFile}></input>
169        <p>{response}</p>
170      </div>
171    </div>
172  );
173}
174
175function SpeechToText(props) {
176  const [response, setResponse] = useState("Press 'start recording' to begin your transcription. Press STOP recording once you finish speaking.")
177  
178  function AudioRecorder(props) {
179    const [recording, setRecording] = useState(false);
180    const [micStream, setMicStream] = useState();
181    const [audioBuffer] = useState(
182      (function() {
183        let buffer = [];
184        function add(raw) {
185          buffer = buffer.concat(...raw);
186          return buffer;
187        }
188        function newBuffer() {
189          console.log("resetting buffer");
190          buffer = [];
191        }
192 
193        return {
194          reset: function() {
195            newBuffer();
196          },
197          addData: function(raw) {
198            return add(raw);
199          },
200          getData: function() {
201            return buffer;
202          }
203        };
204      })()
205    );
206
207    async function startRecording() {
208      console.log('start recording');
209      audioBuffer.reset();
210
211      window.navigator.mediaDevices.getUserMedia({ video: false, audio: true }).then((stream) => {
212        const startMic = new mic();
213
214        startMic.setStream(stream);
215        startMic.on('data', (chunk) => {
216          var raw = mic.toRaw(chunk);
217          if (raw == null) {
218            return;
219          }
220          audioBuffer.addData(raw);
221
222        });
223
224        setRecording(true);
225        setMicStream(startMic);
226      });
227    }
228
229    async function stopRecording() {
230      console.log('stop recording');
231      const { finishRecording } = props;
232
233      micStream.stop();
234      setMicStream(null);
235      setRecording(false);
236
237      const resultBuffer = audioBuffer.getData();
238
239      if (typeof finishRecording === "function") {
240        finishRecording(resultBuffer);
241      }
242
243    }
244
245    return (
246      <div className="audioRecorder">
247        <div>
248          {recording && <button onClick={stopRecording}>Stop recording</button>}
249          {!recording && <button onClick={startRecording}>Start recording</button>}
250        </div>
251      </div>
252    );
253  }
254
255  function convertFromBuffer(bytes) {
256    setResponse('Converting text...');
257    
258    Predictions.convert({
259      transcription: {
260        source: {
261          bytes
262        },
263        // language: "en-US", // other options are "en-GB", "fr-FR", "fr-CA", "es-US"
264      },
265    }).then(({ transcription: { fullText } }) => setResponse(fullText))
266      .catch(err => setResponse(JSON.stringify(err, null, 2)))
267  }
268
269  return (
270    <div className="Text">
271      <div>
272        <h3>Speech to text</h3>
273        <AudioRecorder finishRecording={convertFromBuffer} />
274        <p>{response}</p>
275      </div>
276    </div>
277  );
278}
279
280function TextToSpeech() {
281  const [response, setResponse] = useState("...")
282  const [textToGenerateSpeech, setTextToGenerateSpeech] = useState("write to speech");
283
284  function generateTextToSpeech() {
285    setResponse('Generating audio...');
286    Predictions.convert({
287      textToSpeech: {
288        source: {
289          text: textToGenerateSpeech,
290        },
291        voiceId: "Amy" // default configured on aws-exports.js 
292        // list of different options are here https://docs.aws.amazon.com/polly/latest/dg/voicelist.html
293      }
294    }).then(result => {
295      let AudioContext = window.AudioContext || window.webkitAudioContext;
296      console.log({ AudioContext });
297      const audioCtx = new AudioContext(); 
298      const source = audioCtx.createBufferSource();
299      audioCtx.decodeAudioData(result.audioStream, (buffer) => {
300
301        source.buffer = buffer;
302        source.connect(audioCtx.destination);
303        source.start(0);
304      }, (err) => console.log({err}));
305      
306      setResponse(`Generation completed, press play`);
307    })
308      .catch(err => setResponse(err))
309  }
310
311  function setText(event) {
312    setTextToGenerateSpeech(event.target.value);
313  }
314
315  return (
316    <div className="Text">
317      <div>
318        <h3>Text To Speech</h3>
319        <input value={textToGenerateSpeech} onChange={setText}></input>
320        <button onClick={generateTextToSpeech}>Text to Speech</button>
321        <h3>{response}</h3>
322      </div>
323    </div>
324  );
325}
326
327function TextTranslation() {
328  const [response, setResponse] = useState("Input some text and click enter to test")
329  const [textToTranslate, setTextToTranslate] = useState("write to translate");
330
331  function translate() {
332    Predictions.convert({
333      translateText: {
334        source: {
335          text: textToTranslate,
336          // language : "es" // defaults configured on aws-exports.js
337          // supported languages https://docs.aws.amazon.com/translate/latest/dg/how-it-works.html#how-it-works-language-codes
338        },
339        // targetLanguage: "en"
340      }
341    }).then(result => setResponse(JSON.stringify(result, null, 2)))
342      .catch(err => setResponse(JSON.stringify(err, null, 2)))
343  }
344
345  function setText(event) {
346    setTextToTranslate(event.target.value);
347  }
348
349  return (
350    <div className="Text">
351      <div>
352        <h3>Text Translation</h3>
353        <input value={textToTranslate} onChange={setText}></input>
354        <button onClick={translate}>Translate</button>
355        <p>{response}</p>
356      </div>
357    </div>
358  );
359}
360
361function TextInterpretation() {
362  const [response, setResponse] = useState("Input some text and click enter to test")
363  const [textToInterpret, setTextToInterpret] = useState("write some text here to interpret");
364
365  function interpretFromPredictions() {
366    Predictions.interpret({
367      text: {
368        source: {
369          text: textToInterpret,
370        },
371        type: "ALL"
372      }
373    }).then(result => setResponse(JSON.stringify(result, null, 2)))
374      .catch(err => setResponse(JSON.stringify(err, null, 2)))
375  }
376
377  function setText(event) {
378    setTextToInterpret(event.target.value);
379  }
380
381  return (
382    <div className="Text">
383      <div>
384        <h3>Text interpretation</h3>
385        <input value={textToInterpret} onChange={setText}></input>
386        <button onClick={interpretFromPredictions}>test</button>
387        <p>{response}</p>
388      </div>
389    </div>
390  );
391}
392
393function App() {
394  return (
395    <div className="App">
396      Translate Text
397      <TextTranslation />
398      <br/>
399      Speech Generation
400      <TextToSpeech />
401      <br/>
402      Transcribe Audio
403      <SpeechToText />
404      <br/>
405      Identify Text
406      <TextIdentification />
407      <br/>
408      Identify Entities
409      <EntityIdentification />
410      <br/>
411      Identify Entities (Advanced)
412      <PredictionsUpload />
413      <br/>
414      Label Objects
415      <LabelsIdentification />
416      <br/>
417      Text Interpretation
418      <TextInterpretation />
419    </div>
420  );
421}
422
423export default App;