How to fetch data (Text) from Website in Flutter / dart?
217
From my point of view, you want to create a scraper. I checked the target website (https://www.porsche-914.com/forum), it can be scraped without any special technique (they don't have many Ajax calls (you may test by using Postman)). So it is a possible task. My suggestion flow is:
- Load the raw HTML using any technique (http, dio, inappwebview ...)
- Parse it using BeautifulSoup (https://pub.dev/packages/beautiful_soup_dart) (or any parser; it is just my favorite parser.)
- Map it to your existing model.
Here is some example code. Hope it helps:
import 'package:http/http.dart' as http;
import 'package:beautiful_soup_dart/beautiful_soup.dart';
class TestParse {
excuteSample() async {
var url = Uri.parse('https://www.porsche-914.com/forum');
var response = await http.get(url);
BeautifulSoup bs = BeautifulSoup(response.body);
final allHeaderName = bs.findAll('td', attrs: {'class': 'oben'});
allHeaderName.forEach((element) {
print('the header: ${element.text}');
});
}
}
Here is the result:
The final result you need is a long story and long code. Hope this will give you a starting point.
UPDATE: I added the full demo code, using your requested code:
import 'package:beautiful_soup_dart/beautiful_soup.dart';
import 'package:flutter/material.dart';
import 'package:http/http.dart' as http;
void main() => runApp(const MaterialApp(home: MyApp()));
class MyApp extends StatefulWidget {
const MyApp({Key? key}) : super(key: key);
@override
State<MyApp> createState() => _MyAppState();
}
class _MyAppState extends State<MyApp> {
final testparse = TestParse();
List<String> yourModelPleaseReplaceThis = [];
@override
void initState() {
super.initState();
excuteRequestAndParse();
}
excuteRequestAndParse() async {
final result =
await testparse.excuteSample(); //[1] i guess you missing this await
setState(() {
yourModelPleaseReplaceThis = result;
});
}
@override
Widget build(BuildContext context) {
return ListView.builder(
itemBuilder: (context, index) {
return Center(
child: NeumorphismPost(
title: yourModelPleaseReplaceThis[index],
titleColor: Theme.of(context).cardColor,
textColor: Theme.of(context).cardColor,
text: yourModelPleaseReplaceThis[index],
width: 370,
onTap: () {
//THIS CODE BELOW IS YOUR CODE....
},
),
);
},
itemCount: yourModelPleaseReplaceThis.length,
);
}
}
// BELOW IS TESTING CODE....
class TestParse {
Future<List<String>> excuteSample() async {
var url = Uri.parse('https://www.porsche-914.com/forum');
var response = await http.get(url);
BeautifulSoup bs = BeautifulSoup(response.body);
final allHeaderName = bs.findAll('td', attrs: {'class': 'oben'});
allHeaderName.forEach((element) {
print('the header: ${element.text}');
});
return Future.value(allHeaderName.map((e) => e.text).toList());
}
}
class NeumorphismPost extends StatefulWidget {
final double width;
final double height;
final double borderRadius;
final String title;
final String text;
final Color titleColor;
final Color textColor;
final double titleSize;
final double textSize;
final Function onTap;
NeumorphismPost({
this.width = 185,
this.height = 185,
this.title = "Title",
this.text = "",
this.borderRadius = 20,
this.titleColor = const Color(0xFF424242),
this.textColor = const Color(0xFF424242),
this.titleSize = 22,
this.textSize = 18,
required this.onTap,
});
@override
State<NeumorphismPost> createState() => _NeumorphismPostState();
}
class _NeumorphismPostState extends State<NeumorphismPost> {
@override
Widget build(BuildContext context) {
return Padding(
padding: const EdgeInsets.all(8.0),
child: Wrap(
direction: Axis.vertical,
children: [
SingleChildScrollView(
child: SizedBox(
height: widget.height,
width: widget.width,
child: GestureDetector(
onTap: () {},
child: Scaffold(
backgroundColor: Colors.transparent,
body: SizedBox(
height: widget.height,
width: widget.width,
child: Center(
child: Container(
height: widget.height,
width: widget.width,
decoration: BoxDecoration(
color: Theme.of(context).backgroundColor,
borderRadius:
BorderRadius.circular(widget.borderRadius),
boxShadow: [
BoxShadow(
color: Theme.of(context).hintColor,
offset: const Offset(5, 5),
blurRadius: 15,
spreadRadius: 1,
),
BoxShadow(
color: Theme.of(context).backgroundColor,
offset: Offset(-5, -5),
blurRadius: 15,
spreadRadius: 1,
)
],
),
child: Wrap(
direction: Axis.horizontal,
children: [
SingleChildScrollView(
child: Center(
child: Column(
mainAxisAlignment:
MainAxisAlignment.spaceEvenly,
children: <Widget>[
Wrap(
direction: Axis.horizontal,
children: [
SizedBox(
height: widget.height / 3,
child: Wrap(
children: [
Column(
mainAxisAlignment:
MainAxisAlignment.start,
crossAxisAlignment:
CrossAxisAlignment.start,
children: [
Padding(
padding: const EdgeInsets
.symmetric(
horizontal: 15,
vertical: 15),
child: Text(
widget.title,
textAlign:
TextAlign.center,
overflow:
TextOverflow.fade,
style: TextStyle(
fontSize:
widget.titleSize,
fontWeight:
FontWeight.bold,
color:
widget.titleColor,
),
),
),
],
),
],
),
),
],
),
Wrap(
direction: Axis.horizontal,
children: [
GestureDetector(
onTap: () {
widget.onTap();
},
child: Padding(
padding: const EdgeInsets.symmetric(
vertical: 5),
child: SizedBox(
height: widget.height / 1.38,
child: Padding(
padding:
const EdgeInsets.symmetric(
horizontal: 15,
vertical: 15),
child: Text(
widget.text,
textAlign: TextAlign.center,
style: TextStyle(
fontSize: widget.textSize,
fontWeight:
FontWeight.normal,
color: widget.textColor,
),
),
),
),
),
),
],
),
],
),
),
),
],
),
),
),
),
),
),
),
),
],
),
);
}
}
This is the result:
Please note:
- This list I created is for sample, so it is just a list of Strings. You should create a complete Model.
Author by
Acidic
Updated on January 04, 2023Comments
-
Acidic over 1 year
So, I'm pretty new on Flutter (just about a week) and dart but I hope this Question is not toooooo stupid:
I'm trying to create an App for an existing Website (a forum). I dont want to use the flutter WebView, because I want to create an comepletly new Interface (The Website is not optimized for mobile View, that's why I'm builing this App).
The (german) Forum: https://www.porsche-914.com/forum
I want to fetch the single forum topics as well as the posts itself including username, title and date and then map the text to a custom widget to display it. The Widget Part is no problem, it's already done but I cant figure out how to fetch that data from the website and how to store it.
I don't know, hope you can help me...
Thanks a lot for the taken time.
How the mapping should work:
fetched[index].title, fetched[index].text Center( child: NeumorphismPost( title: fetched[index].title, titleColor: Theme.of(context).cardColor, textColor: Theme.of(context).cardColor, text: fetched[index].text, width: 370, onTap: () { _popupPage(context, title: fetched[index].title, child: Padding( padding: const EdgeInsets.all(15.0), child: Text( fetched[index].text, style: TextStyle( color: Theme.of(context).cardColor, fontSize: 18), ), ), ); }, ), ),This is my custom Widget:
import 'package:flutter/material.dart'; class NeumorphismPost extends StatefulWidget { final double width; final double height; final double borderRadius; final String title; final String text; final Color titleColor; final Color textColor; final double titleSize; final double textSize; final Function onTap; NeumorphismPost({ this.width = 185, this.height = 185, this.title = "Title", this.text = "", this.borderRadius = 20, this.titleColor = const Color(0xFF424242), this.textColor = const Color(0xFF424242), this.titleSize = 22, this.textSize = 18, required this.onTap, }); @override State<NeumorphismPost> createState() => _NeumorphismPostState(); } class _NeumorphismPostState extends State<NeumorphismPost> { @override Widget build(BuildContext context) { return Padding( padding: const EdgeInsets.all(8.0), child: Wrap( direction: Axis.vertical, children: [ SingleChildScrollView( child: SizedBox( height: widget.height, width: widget.width, child: GestureDetector( onTap: () {}, child: Scaffold( backgroundColor: Colors.transparent, body: SizedBox( height: widget.height, width: widget.width, child: Center( child: Container( height: widget.height, width: widget.width, decoration: BoxDecoration( color: Theme.of(context).backgroundColor, borderRadius: BorderRadius.circular(widget.borderRadius), boxShadow: [ BoxShadow( color: Theme.of(context).hintColor, offset: const Offset(5, 5), blurRadius: 15, spreadRadius: 1, ), BoxShadow( color: Theme.of(context).backgroundColor, offset: Offset(-5, -5), blurRadius: 15, spreadRadius: 1, ) ], ), child: Wrap( direction: Axis.horizontal, children: [ SingleChildScrollView( child: Center( child: Column( mainAxisAlignment: MainAxisAlignment.spaceEvenly, children: <Widget>[ Wrap( direction: Axis.horizontal, children: [ SizedBox( height: widget.height / 3, child: Wrap( children: [ Column( mainAxisAlignment: MainAxisAlignment.start, crossAxisAlignment: CrossAxisAlignment.start, children: [ Padding( padding: const EdgeInsets .symmetric( horizontal: 15, vertical: 15), child: Text( widget.title, textAlign: TextAlign.center, overflow: TextOverflow.fade, style: TextStyle( fontSize: widget.titleSize, fontWeight: FontWeight.bold, color: widget.titleColor, ), ), ), ], ), ], ), ), ], ), Wrap( direction: Axis.horizontal, children: [ GestureDetector( onTap: () { widget.onTap(); }, child: Padding( padding: const EdgeInsets.symmetric( vertical: 5), child: SizedBox( height: widget.height / 1.38, child: Padding( padding: const EdgeInsets.symmetric( horizontal: 15, vertical: 15), child: Text( widget.text, textAlign: TextAlign.center, style: TextStyle( fontSize: widget.textSize, fontWeight: FontWeight.normal, color: widget.textColor, ), ), ), ), ), ), ], ), ], ), ), ), ], ), ), ), ), ), ), ), ), ], ), ); } } -
Acidic about 2 yearsOk, that was quick, thank you so much, that's a great base to build on. I got now another problemt tho, because as I try to map everything on a Widget I get the error message 'type 'Future<dynamic>' is not a subtype of type 'Widget' in type cast'... Any idea how to fix that?
-
nielsezeka about 2 yearsI guess it may be a problem with your async/await, so I updated the full demo(using your code), It so long and I can't paste in comment, so please check section "UPDATE" in original post. Hope it help!
